2024-11-19 00:13:08,457 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-19 00:13:08,469 main DEBUG Took 0.009382 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-19 00:13:08,469 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-19 00:13:08,469 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-19 00:13:08,470 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-19 00:13:08,471 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:08,478 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-19 00:13:08,489 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:08,491 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:08,491 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:08,491 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:08,492 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:08,492 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:08,493 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:08,493 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:08,494 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:08,494 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:08,495 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:08,495 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:08,495 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:08,496 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:08,496 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:08,496 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:08,497 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:08,497 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:08,497 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:08,498 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:08,498 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:08,498 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:08,499 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:08,499 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 00:13:08,500 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:08,500 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-19 00:13:08,501 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 00:13:08,503 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-19 00:13:08,504 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-19 00:13:08,505 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-19 00:13:08,506 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-19 00:13:08,506 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-19 00:13:08,514 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-19 00:13:08,516 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-19 00:13:08,518 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-19 00:13:08,518 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-19 00:13:08,519 main DEBUG createAppenders(={Console}) 2024-11-19 00:13:08,519 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-19 00:13:08,520 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-19 00:13:08,520 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-19 00:13:08,520 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-19 00:13:08,521 main DEBUG OutputStream closed 2024-11-19 00:13:08,521 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-19 00:13:08,521 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-19 00:13:08,521 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-19 00:13:08,589 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-19 00:13:08,592 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-19 00:13:08,593 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-19 00:13:08,593 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-19 00:13:08,594 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-19 00:13:08,594 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-19 00:13:08,595 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-19 00:13:08,595 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-19 00:13:08,596 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-19 00:13:08,596 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-19 00:13:08,596 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-19 00:13:08,596 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-19 00:13:08,597 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-19 00:13:08,597 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-19 00:13:08,597 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-19 00:13:08,598 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-19 00:13:08,598 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-19 00:13:08,599 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-19 00:13:08,601 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19 00:13:08,601 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-19 00:13:08,601 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-19 00:13:08,602 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-19T00:13:08,835 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1 2024-11-19 00:13:08,838 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-19 00:13:08,839 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19T00:13:08,848 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-19T00:13:08,871 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-19T00:13:08,893 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/cluster_ec2aa8be-d8f2-2fc7-fbd3-add83417a4c1, deleteOnExit=true 2024-11-19T00:13:08,894 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-19T00:13:08,895 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/test.cache.data in system properties and HBase conf 2024-11-19T00:13:08,896 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/hadoop.tmp.dir in system properties and HBase conf 2024-11-19T00:13:08,897 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/hadoop.log.dir in system properties and HBase conf 2024-11-19T00:13:08,898 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-19T00:13:08,899 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-19T00:13:08,899 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-19T00:13:09,017 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-19T00:13:09,168 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T00:13:09,177 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-19T00:13:09,178 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-19T00:13:09,179 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-19T00:13:09,179 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T00:13:09,180 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-19T00:13:09,182 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-19T00:13:09,183 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T00:13:09,183 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T00:13:09,184 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-19T00:13:09,184 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/nfs.dump.dir in system properties and HBase conf 2024-11-19T00:13:09,185 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/java.io.tmpdir in system properties and HBase conf 2024-11-19T00:13:09,185 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T00:13:09,186 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-19T00:13:09,187 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-19T00:13:10,554 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-19T00:13:10,634 INFO [Time-limited test {}] log.Log(170): Logging initialized @2898ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-19T00:13:10,705 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T00:13:10,770 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T00:13:10,789 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T00:13:10,789 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T00:13:10,790 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-19T00:13:10,801 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T00:13:10,804 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/hadoop.log.dir/,AVAILABLE} 2024-11-19T00:13:10,804 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T00:13:10,975 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/java.io.tmpdir/jetty-localhost-45887-hadoop-hdfs-3_4_1-tests_jar-_-any-12251920631637173650/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T00:13:10,982 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:45887} 2024-11-19T00:13:10,982 INFO [Time-limited test {}] server.Server(415): Started @3248ms 2024-11-19T00:13:11,488 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T00:13:11,495 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T00:13:11,496 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T00:13:11,496 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T00:13:11,496 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-19T00:13:11,497 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/hadoop.log.dir/,AVAILABLE} 2024-11-19T00:13:11,498 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T00:13:11,596 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/java.io.tmpdir/jetty-localhost-41117-hadoop-hdfs-3_4_1-tests_jar-_-any-4082371263417294443/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T00:13:11,597 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:41117} 2024-11-19T00:13:11,597 INFO [Time-limited test {}] server.Server(415): Started @3862ms 2024-11-19T00:13:11,646 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-19T00:13:12,484 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/cluster_ec2aa8be-d8f2-2fc7-fbd3-add83417a4c1/data/data2/current/BP-1176150126-172.17.0.2-1731975190061/current, will proceed with Du for space computation calculation, 2024-11-19T00:13:12,484 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/cluster_ec2aa8be-d8f2-2fc7-fbd3-add83417a4c1/data/data1/current/BP-1176150126-172.17.0.2-1731975190061/current, will proceed with Du for space computation calculation, 2024-11-19T00:13:12,512 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-19T00:13:12,553 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x532dca9474494af3 with lease ID 0xc755bcc944dca74d: Processing first storage report for DS-175ee38d-49e6-4ae7-b658-13d9ba8bd05a from datanode DatanodeRegistration(127.0.0.1:33587, datanodeUuid=7cb7ba0a-ea0f-4bdc-bc40-7cf0d89b01ba, infoPort=35731, infoSecurePort=0, ipcPort=35537, storageInfo=lv=-57;cid=testClusterID;nsid=667369603;c=1731975190061) 2024-11-19T00:13:12,554 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x532dca9474494af3 with lease ID 0xc755bcc944dca74d: from storage DS-175ee38d-49e6-4ae7-b658-13d9ba8bd05a node DatanodeRegistration(127.0.0.1:33587, datanodeUuid=7cb7ba0a-ea0f-4bdc-bc40-7cf0d89b01ba, infoPort=35731, infoSecurePort=0, ipcPort=35537, storageInfo=lv=-57;cid=testClusterID;nsid=667369603;c=1731975190061), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-19T00:13:12,555 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x532dca9474494af3 with lease ID 0xc755bcc944dca74d: Processing first storage report for DS-39d100f6-6b99-4ec0-bfaa-1eeadf40ad35 from datanode DatanodeRegistration(127.0.0.1:33587, datanodeUuid=7cb7ba0a-ea0f-4bdc-bc40-7cf0d89b01ba, infoPort=35731, infoSecurePort=0, ipcPort=35537, storageInfo=lv=-57;cid=testClusterID;nsid=667369603;c=1731975190061) 2024-11-19T00:13:12,555 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x532dca9474494af3 with lease ID 0xc755bcc944dca74d: from storage DS-39d100f6-6b99-4ec0-bfaa-1eeadf40ad35 node DatanodeRegistration(127.0.0.1:33587, datanodeUuid=7cb7ba0a-ea0f-4bdc-bc40-7cf0d89b01ba, infoPort=35731, infoSecurePort=0, ipcPort=35537, storageInfo=lv=-57;cid=testClusterID;nsid=667369603;c=1731975190061), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-19T00:13:12,653 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1 2024-11-19T00:13:12,756 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/cluster_ec2aa8be-d8f2-2fc7-fbd3-add83417a4c1/zookeeper_0, clientPort=62652, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/cluster_ec2aa8be-d8f2-2fc7-fbd3-add83417a4c1/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/cluster_ec2aa8be-d8f2-2fc7-fbd3-add83417a4c1/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-19T00:13:12,766 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62652 2024-11-19T00:13:12,778 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T00:13:12,782 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T00:13:13,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741825_1001 (size=7) 2024-11-19T00:13:13,419 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2 with version=8 2024-11-19T00:13:13,420 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/hbase-staging 2024-11-19T00:13:13,500 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-19T00:13:13,725 INFO [Time-limited test {}] client.ConnectionUtils(128): master/6b1531871d35:0 server-side Connection retries=45 2024-11-19T00:13:13,735 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T00:13:13,736 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T00:13:13,741 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T00:13:13,741 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T00:13:13,741 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T00:13:13,872 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-19T00:13:13,926 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-19T00:13:13,935 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-19T00:13:13,938 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T00:13:13,964 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 2877 (auto-detected) 2024-11-19T00:13:13,965 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-19T00:13:13,985 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33843 2024-11-19T00:13:14,012 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33843 connecting to ZooKeeper ensemble=127.0.0.1:62652 2024-11-19T00:13:14,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:338430x0, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T00:13:14,116 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33843-0x10055461f5a0000 connected 2024-11-19T00:13:14,202 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T00:13:14,205 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T00:13:14,216 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T00:13:14,220 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2, hbase.cluster.distributed=false 2024-11-19T00:13:14,242 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T00:13:14,246 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33843 2024-11-19T00:13:14,247 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33843 2024-11-19T00:13:14,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33843 2024-11-19T00:13:14,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33843 2024-11-19T00:13:14,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33843 2024-11-19T00:13:14,351 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6b1531871d35:0 server-side Connection retries=45 2024-11-19T00:13:14,352 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T00:13:14,353 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T00:13:14,353 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T00:13:14,353 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T00:13:14,353 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T00:13:14,356 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T00:13:14,359 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T00:13:14,360 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38567 2024-11-19T00:13:14,362 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38567 connecting to ZooKeeper ensemble=127.0.0.1:62652 2024-11-19T00:13:14,363 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T00:13:14,366 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T00:13:14,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:385670x0, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T00:13:14,382 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:385670x0, quorum=127.0.0.1:62652, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T00:13:14,382 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38567-0x10055461f5a0001 connected 2024-11-19T00:13:14,386 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T00:13:14,394 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T00:13:14,396 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38567-0x10055461f5a0001, quorum=127.0.0.1:62652, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T00:13:14,402 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38567-0x10055461f5a0001, quorum=127.0.0.1:62652, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T00:13:14,404 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38567 2024-11-19T00:13:14,404 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38567 2024-11-19T00:13:14,406 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38567 2024-11-19T00:13:14,407 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38567 2024-11-19T00:13:14,408 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38567 2024-11-19T00:13:14,423 DEBUG [M:0;6b1531871d35:33843 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;6b1531871d35:33843 2024-11-19T00:13:14,424 INFO [master/6b1531871d35:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/6b1531871d35,33843,1731975193570 2024-11-19T00:13:14,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38567-0x10055461f5a0001, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T00:13:14,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T00:13:14,442 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/6b1531871d35,33843,1731975193570 2024-11-19T00:13:14,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:14,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38567-0x10055461f5a0001, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T00:13:14,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38567-0x10055461f5a0001, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:14,473 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T00:13:14,474 INFO [master/6b1531871d35:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/6b1531871d35,33843,1731975193570 from backup master directory 2024-11-19T00:13:14,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/6b1531871d35,33843,1731975193570 2024-11-19T00:13:14,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38567-0x10055461f5a0001, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T00:13:14,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T00:13:14,484 WARN [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T00:13:14,484 INFO [master/6b1531871d35:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=6b1531871d35,33843,1731975193570 2024-11-19T00:13:14,486 INFO [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-19T00:13:14,488 INFO [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-19T00:13:14,542 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/hbase.id] with ID: 0b318443-22f9-457d-92e4-12357fd10a8d 2024-11-19T00:13:14,542 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/.tmp/hbase.id 2024-11-19T00:13:14,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741826_1002 (size=42) 2024-11-19T00:13:14,957 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/.tmp/hbase.id]:[hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/hbase.id] 2024-11-19T00:13:15,012 INFO [master/6b1531871d35:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T00:13:15,017 INFO [master/6b1531871d35:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-19T00:13:15,036 INFO [master/6b1531871d35:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-19T00:13:15,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:15,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38567-0x10055461f5a0001, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:15,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741827_1003 (size=196) 2024-11-19T00:13:15,110 INFO [master/6b1531871d35:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T00:13:15,112 INFO [master/6b1531871d35:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-19T00:13:15,127 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:15,131 INFO [master/6b1531871d35:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T00:13:15,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741828_1004 (size=1189) 2024-11-19T00:13:15,583 INFO [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/data/master/store 2024-11-19T00:13:15,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741829_1005 (size=34) 2024-11-19T00:13:16,008 INFO [master/6b1531871d35:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-19T00:13:16,012 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:13:16,013 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T00:13:16,013 INFO [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T00:13:16,014 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T00:13:16,015 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T00:13:16,016 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T00:13:16,016 INFO [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T00:13:16,017 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731975196013Disabling compacts and flushes for region at 1731975196013Disabling writes for close at 1731975196015 (+2 ms)Writing region close event to WAL at 1731975196016 (+1 ms)Closed at 1731975196016 2024-11-19T00:13:16,020 WARN [master/6b1531871d35:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/data/master/store/.initializing 2024-11-19T00:13:16,021 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/WALs/6b1531871d35,33843,1731975193570 2024-11-19T00:13:16,032 INFO [master/6b1531871d35:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T00:13:16,047 INFO [master/6b1531871d35:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6b1531871d35%2C33843%2C1731975193570, suffix=, logDir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/WALs/6b1531871d35,33843,1731975193570, archiveDir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/oldWALs, maxLogs=10 2024-11-19T00:13:16,067 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/WALs/6b1531871d35,33843,1731975193570/6b1531871d35%2C33843%2C1731975193570.1731975196051, exclude list is [], retry=0 2024-11-19T00:13:16,082 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33587,DS-175ee38d-49e6-4ae7-b658-13d9ba8bd05a,DISK] 2024-11-19T00:13:16,085 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-19T00:13:16,117 INFO [master/6b1531871d35:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/WALs/6b1531871d35,33843,1731975193570/6b1531871d35%2C33843%2C1731975193570.1731975196051 2024-11-19T00:13:16,117 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35731:35731)] 2024-11-19T00:13:16,118 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-19T00:13:16,118 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:13:16,121 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T00:13:16,122 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T00:13:16,160 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T00:13:16,184 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-19T00:13:16,188 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:16,191 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T00:13:16,191 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T00:13:16,195 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-19T00:13:16,195 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:16,197 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:13:16,197 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T00:13:16,200 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-19T00:13:16,200 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:16,201 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:13:16,202 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T00:13:16,205 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-19T00:13:16,205 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:16,206 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:13:16,206 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T00:13:16,210 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T00:13:16,211 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T00:13:16,216 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T00:13:16,217 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T00:13:16,221 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T00:13:16,224 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T00:13:16,229 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T00:13:16,230 INFO [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65224646, jitterRate=-0.028077036142349243}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T00:13:16,237 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731975196133Initializing all the Stores at 1731975196135 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731975196136 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975196136Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975196136Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975196137 (+1 ms)Cleaning up temporary data from old regions at 1731975196217 (+80 ms)Region opened successfully at 1731975196236 (+19 ms) 2024-11-19T00:13:16,238 INFO [master/6b1531871d35:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-19T00:13:16,270 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2c71b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6b1531871d35/172.17.0.2:0 2024-11-19T00:13:16,299 INFO [master/6b1531871d35:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-19T00:13:16,309 INFO [master/6b1531871d35:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-19T00:13:16,310 INFO [master/6b1531871d35:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-19T00:13:16,312 INFO [master/6b1531871d35:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T00:13:16,314 INFO [master/6b1531871d35:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-19T00:13:16,319 INFO [master/6b1531871d35:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-19T00:13:16,319 INFO [master/6b1531871d35:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-19T00:13:16,345 INFO [master/6b1531871d35:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-19T00:13:16,353 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-19T00:13:16,388 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-19T00:13:16,392 INFO [master/6b1531871d35:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-19T00:13:16,394 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-19T00:13:16,405 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-19T00:13:16,408 INFO [master/6b1531871d35:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-19T00:13:16,413 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-19T00:13:16,424 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-19T00:13:16,427 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-19T00:13:16,433 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-19T00:13:16,453 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-19T00:13:16,463 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-19T00:13:16,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T00:13:16,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38567-0x10055461f5a0001, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T00:13:16,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38567-0x10055461f5a0001, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:16,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:16,479 INFO [master/6b1531871d35:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=6b1531871d35,33843,1731975193570, sessionid=0x10055461f5a0000, setting cluster-up flag (Was=false) 2024-11-19T00:13:16,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:16,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38567-0x10055461f5a0001, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:16,533 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-19T00:13:16,541 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6b1531871d35,33843,1731975193570 2024-11-19T00:13:16,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:16,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38567-0x10055461f5a0001, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:16,591 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-19T00:13:16,593 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6b1531871d35,33843,1731975193570 2024-11-19T00:13:16,602 INFO [master/6b1531871d35:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-19T00:13:16,614 INFO [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(746): ClusterId : 0b318443-22f9-457d-92e4-12357fd10a8d 2024-11-19T00:13:16,617 DEBUG [RS:0;6b1531871d35:38567 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T00:13:16,627 DEBUG [RS:0;6b1531871d35:38567 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T00:13:16,627 DEBUG [RS:0;6b1531871d35:38567 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T00:13:16,634 DEBUG [RS:0;6b1531871d35:38567 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T00:13:16,635 DEBUG [RS:0;6b1531871d35:38567 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@557afaf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6b1531871d35/172.17.0.2:0 2024-11-19T00:13:16,653 DEBUG [RS:0;6b1531871d35:38567 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;6b1531871d35:38567 2024-11-19T00:13:16,657 INFO [RS:0;6b1531871d35:38567 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T00:13:16,657 INFO [RS:0;6b1531871d35:38567 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T00:13:16,657 DEBUG [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T00:13:16,661 INFO [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(2659): reportForDuty to master=6b1531871d35,33843,1731975193570 with port=38567, startcode=1731975194317 2024-11-19T00:13:16,675 DEBUG [RS:0;6b1531871d35:38567 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T00:13:16,689 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T00:13:16,700 INFO [master/6b1531871d35:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-19T00:13:16,707 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38737, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T00:13:16,708 INFO [master/6b1531871d35:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-19T00:13:16,714 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33843 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:16,715 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 6b1531871d35,33843,1731975193570 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-19T00:13:16,725 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/6b1531871d35:0, corePoolSize=5, maxPoolSize=5 2024-11-19T00:13:16,725 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/6b1531871d35:0, corePoolSize=5, maxPoolSize=5 2024-11-19T00:13:16,725 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/6b1531871d35:0, corePoolSize=5, maxPoolSize=5 2024-11-19T00:13:16,726 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/6b1531871d35:0, corePoolSize=5, maxPoolSize=5 2024-11-19T00:13:16,726 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/6b1531871d35:0, corePoolSize=10, maxPoolSize=10 2024-11-19T00:13:16,726 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:16,726 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/6b1531871d35:0, corePoolSize=2, maxPoolSize=2 2024-11-19T00:13:16,727 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:16,730 INFO [master/6b1531871d35:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731975226730 2024-11-19T00:13:16,731 INFO [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-19T00:13:16,732 INFO [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-19T00:13:16,734 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T00:13:16,734 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-19T00:13:16,736 INFO [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-19T00:13:16,736 INFO [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-19T00:13:16,737 INFO [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-19T00:13:16,737 INFO [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-19T00:13:16,738 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:16,741 INFO [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-19T00:13:16,742 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:16,742 INFO [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-19T00:13:16,743 INFO [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-19T00:13:16,743 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-19T00:13:16,745 INFO [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-19T00:13:16,745 INFO [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-19T00:13:16,747 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/6b1531871d35:0:becomeActiveMaster-HFileCleaner.large.0-1731975196747,5,FailOnTimeoutGroup] 2024-11-19T00:13:16,747 DEBUG [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-19T00:13:16,748 WARN [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-19T00:13:16,748 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/6b1531871d35:0:becomeActiveMaster-HFileCleaner.small.0-1731975196747,5,FailOnTimeoutGroup] 2024-11-19T00:13:16,748 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:16,748 INFO [master/6b1531871d35:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-19T00:13:16,750 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:16,750 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:16,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741831_1007 (size=1321) 2024-11-19T00:13:16,851 INFO [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(2659): reportForDuty to master=6b1531871d35,33843,1731975193570 with port=38567, startcode=1731975194317 2024-11-19T00:13:16,857 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33843 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6b1531871d35,38567,1731975194317 2024-11-19T00:13:16,860 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33843 {}] master.ServerManager(517): Registering regionserver=6b1531871d35,38567,1731975194317 2024-11-19T00:13:16,868 DEBUG [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2 2024-11-19T00:13:16,868 DEBUG [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36755 2024-11-19T00:13:16,869 DEBUG [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T00:13:16,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T00:13:16,881 DEBUG [RS:0;6b1531871d35:38567 {}] zookeeper.ZKUtil(111): regionserver:38567-0x10055461f5a0001, quorum=127.0.0.1:62652, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6b1531871d35,38567,1731975194317 2024-11-19T00:13:16,881 WARN [RS:0;6b1531871d35:38567 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T00:13:16,881 INFO [RS:0;6b1531871d35:38567 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T00:13:16,882 DEBUG [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/WALs/6b1531871d35,38567,1731975194317 2024-11-19T00:13:16,883 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6b1531871d35,38567,1731975194317] 2024-11-19T00:13:16,909 INFO [RS:0;6b1531871d35:38567 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T00:13:16,925 INFO [RS:0;6b1531871d35:38567 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T00:13:16,929 INFO [RS:0;6b1531871d35:38567 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T00:13:16,930 INFO [RS:0;6b1531871d35:38567 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:16,931 INFO [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T00:13:16,936 INFO [RS:0;6b1531871d35:38567 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T00:13:16,937 INFO [RS:0;6b1531871d35:38567 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:16,937 DEBUG [RS:0;6b1531871d35:38567 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:16,937 DEBUG [RS:0;6b1531871d35:38567 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:16,938 DEBUG [RS:0;6b1531871d35:38567 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:16,938 DEBUG [RS:0;6b1531871d35:38567 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:16,938 DEBUG [RS:0;6b1531871d35:38567 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:16,938 DEBUG [RS:0;6b1531871d35:38567 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6b1531871d35:0, corePoolSize=2, maxPoolSize=2 2024-11-19T00:13:16,938 DEBUG [RS:0;6b1531871d35:38567 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:16,938 DEBUG [RS:0;6b1531871d35:38567 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:16,938 DEBUG [RS:0;6b1531871d35:38567 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:16,939 DEBUG [RS:0;6b1531871d35:38567 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:16,939 DEBUG [RS:0;6b1531871d35:38567 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:16,939 DEBUG [RS:0;6b1531871d35:38567 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6b1531871d35:0, corePoolSize=1, maxPoolSize=1 2024-11-19T00:13:16,939 DEBUG [RS:0;6b1531871d35:38567 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6b1531871d35:0, corePoolSize=3, maxPoolSize=3 2024-11-19T00:13:16,939 DEBUG [RS:0;6b1531871d35:38567 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0, corePoolSize=3, maxPoolSize=3 2024-11-19T00:13:16,940 INFO [RS:0;6b1531871d35:38567 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:16,940 INFO [RS:0;6b1531871d35:38567 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:16,941 INFO [RS:0;6b1531871d35:38567 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:16,941 INFO [RS:0;6b1531871d35:38567 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:16,941 INFO [RS:0;6b1531871d35:38567 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:16,941 INFO [RS:0;6b1531871d35:38567 {}] hbase.ChoreService(168): Chore ScheduledChore name=6b1531871d35,38567,1731975194317-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T00:13:16,961 INFO [RS:0;6b1531871d35:38567 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T00:13:16,963 INFO [RS:0;6b1531871d35:38567 {}] hbase.ChoreService(168): Chore ScheduledChore name=6b1531871d35,38567,1731975194317-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:16,963 INFO [RS:0;6b1531871d35:38567 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:16,963 INFO [RS:0;6b1531871d35:38567 {}] regionserver.Replication(171): 6b1531871d35,38567,1731975194317 started 2024-11-19T00:13:16,980 INFO [RS:0;6b1531871d35:38567 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:16,981 INFO [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(1482): Serving as 6b1531871d35,38567,1731975194317, RpcServer on 6b1531871d35/172.17.0.2:38567, sessionid=0x10055461f5a0001 2024-11-19T00:13:16,982 DEBUG [RS:0;6b1531871d35:38567 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T00:13:16,982 DEBUG [RS:0;6b1531871d35:38567 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6b1531871d35,38567,1731975194317 2024-11-19T00:13:16,982 DEBUG [RS:0;6b1531871d35:38567 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6b1531871d35,38567,1731975194317' 2024-11-19T00:13:16,982 DEBUG [RS:0;6b1531871d35:38567 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T00:13:16,983 DEBUG [RS:0;6b1531871d35:38567 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T00:13:16,984 DEBUG [RS:0;6b1531871d35:38567 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T00:13:16,984 DEBUG [RS:0;6b1531871d35:38567 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T00:13:16,984 DEBUG [RS:0;6b1531871d35:38567 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6b1531871d35,38567,1731975194317 2024-11-19T00:13:16,984 DEBUG [RS:0;6b1531871d35:38567 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6b1531871d35,38567,1731975194317' 2024-11-19T00:13:16,984 DEBUG [RS:0;6b1531871d35:38567 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T00:13:16,985 DEBUG [RS:0;6b1531871d35:38567 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T00:13:16,986 DEBUG [RS:0;6b1531871d35:38567 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T00:13:16,986 INFO [RS:0;6b1531871d35:38567 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T00:13:16,986 INFO [RS:0;6b1531871d35:38567 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T00:13:17,093 INFO [RS:0;6b1531871d35:38567 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T00:13:17,097 INFO [RS:0;6b1531871d35:38567 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6b1531871d35%2C38567%2C1731975194317, suffix=, logDir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/WALs/6b1531871d35,38567,1731975194317, archiveDir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/oldWALs, maxLogs=32 2024-11-19T00:13:17,121 DEBUG [RS:0;6b1531871d35:38567 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/WALs/6b1531871d35,38567,1731975194317/6b1531871d35%2C38567%2C1731975194317.1731975197100, exclude list is [], retry=0 2024-11-19T00:13:17,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33587,DS-175ee38d-49e6-4ae7-b658-13d9ba8bd05a,DISK] 2024-11-19T00:13:17,131 INFO [RS:0;6b1531871d35:38567 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/WALs/6b1531871d35,38567,1731975194317/6b1531871d35%2C38567%2C1731975194317.1731975197100 2024-11-19T00:13:17,132 DEBUG [RS:0;6b1531871d35:38567 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35731:35731)] 2024-11-19T00:13:17,156 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-19T00:13:17,157 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2 2024-11-19T00:13:17,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741833_1009 (size=32) 2024-11-19T00:13:17,575 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:13:17,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T00:13:17,582 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T00:13:17,582 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:17,583 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T00:13:17,583 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T00:13:17,585 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T00:13:17,586 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:17,587 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T00:13:17,587 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T00:13:17,589 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T00:13:17,589 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:17,590 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T00:13:17,590 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T00:13:17,593 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T00:13:17,593 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:17,594 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T00:13:17,594 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T00:13:17,595 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740 2024-11-19T00:13:17,596 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740 2024-11-19T00:13:17,599 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T00:13:17,599 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T00:13:17,600 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T00:13:17,603 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T00:13:17,607 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T00:13:17,608 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69789177, jitterRate=0.03993977606296539}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T00:13:17,611 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731975197575Initializing all the Stores at 1731975197579 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731975197579Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731975197579Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975197579Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731975197579Cleaning up temporary data from old regions at 1731975197599 (+20 ms)Region opened successfully at 1731975197611 (+12 ms) 2024-11-19T00:13:17,611 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T00:13:17,611 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T00:13:17,611 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T00:13:17,611 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T00:13:17,611 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T00:13:17,613 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T00:13:17,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731975197611Disabling compacts and flushes for region at 1731975197611Disabling writes for close at 1731975197611Writing region close event to WAL at 1731975197612 (+1 ms)Closed at 1731975197613 (+1 ms) 2024-11-19T00:13:17,616 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T00:13:17,616 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-19T00:13:17,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T00:13:17,629 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T00:13:17,632 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-19T00:13:17,786 DEBUG [6b1531871d35:33843 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-19T00:13:17,802 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:13:17,811 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6b1531871d35,38567,1731975194317, state=OPENING 2024-11-19T00:13:17,855 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-19T00:13:17,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38567-0x10055461f5a0001, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:17,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:13:17,867 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T00:13:17,867 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T00:13:17,869 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T00:13:17,872 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6b1531871d35,38567,1731975194317}] 2024-11-19T00:13:18,053 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T00:13:18,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58947, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T00:13:18,073 INFO [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T00:13:18,073 INFO [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T00:13:18,074 INFO [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T00:13:18,079 INFO [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6b1531871d35%2C38567%2C1731975194317.meta, suffix=.meta, logDir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/WALs/6b1531871d35,38567,1731975194317, archiveDir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/oldWALs, maxLogs=32 2024-11-19T00:13:18,107 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/WALs/6b1531871d35,38567,1731975194317/6b1531871d35%2C38567%2C1731975194317.meta.1731975198082.meta, exclude list is [], retry=0 2024-11-19T00:13:18,115 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33587,DS-175ee38d-49e6-4ae7-b658-13d9ba8bd05a,DISK] 2024-11-19T00:13:18,121 INFO [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/WALs/6b1531871d35,38567,1731975194317/6b1531871d35%2C38567%2C1731975194317.meta.1731975198082.meta 2024-11-19T00:13:18,122 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35731:35731)] 2024-11-19T00:13:18,123 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T00:13:18,126 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T00:13:18,130 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T00:13:18,136 INFO [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T00:13:18,143 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T00:13:18,144 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:13:18,145 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T00:13:18,145 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T00:13:18,150 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T00:13:18,153 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T00:13:18,153 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:18,155 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T00:13:18,157 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T00:13:18,161 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T00:13:18,161 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:18,162 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T00:13:18,163 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T00:13:18,165 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T00:13:18,165 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:18,167 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T00:13:18,167 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T00:13:18,169 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T00:13:18,170 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:18,171 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T00:13:18,171 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T00:13:18,173 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740 2024-11-19T00:13:18,178 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740 2024-11-19T00:13:18,181 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T00:13:18,181 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T00:13:18,182 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T00:13:18,186 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T00:13:18,188 INFO [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69793388, jitterRate=0.040002524852752686}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T00:13:18,189 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T00:13:18,191 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731975198146Writing region info on filesystem at 1731975198146Initializing all the Stores at 1731975198149 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731975198149Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731975198150 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975198150Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731975198150Cleaning up temporary data from old regions at 1731975198181 (+31 ms)Running coprocessor post-open hooks at 1731975198189 (+8 ms)Region opened successfully at 1731975198191 (+2 ms) 2024-11-19T00:13:18,201 INFO [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731975198042 2024-11-19T00:13:18,219 DEBUG [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T00:13:18,219 INFO [RS_OPEN_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T00:13:18,223 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:13:18,227 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6b1531871d35,38567,1731975194317, state=OPEN 2024-11-19T00:13:18,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38567-0x10055461f5a0001, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T00:13:18,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T00:13:18,318 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T00:13:18,318 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T00:13:18,319 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:18,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-19T00:13:18,334 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6b1531871d35,38567,1731975194317 in 447 msec 2024-11-19T00:13:18,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-19T00:13:18,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 714 msec 2024-11-19T00:13:18,351 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T00:13:18,352 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-19T00:13:18,378 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:13:18,379 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:13:18,406 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:13:18,409 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44523, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:13:18,442 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7990 sec 2024-11-19T00:13:18,443 INFO [master/6b1531871d35:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731975198442, completionTime=-1 2024-11-19T00:13:18,446 INFO [master/6b1531871d35:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-19T00:13:18,447 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T00:13:18,482 INFO [master/6b1531871d35:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-19T00:13:18,483 INFO [master/6b1531871d35:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731975258482 2024-11-19T00:13:18,483 INFO [master/6b1531871d35:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731975318483 2024-11-19T00:13:18,483 INFO [master/6b1531871d35:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 36 msec 2024-11-19T00:13:18,486 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6b1531871d35,33843,1731975193570-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:18,486 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6b1531871d35,33843,1731975193570-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:18,486 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6b1531871d35,33843,1731975193570-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:18,488 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-6b1531871d35:33843, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:18,488 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:18,489 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:18,498 DEBUG [master/6b1531871d35:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-19T00:13:18,534 INFO [master/6b1531871d35:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.050sec 2024-11-19T00:13:18,536 INFO [master/6b1531871d35:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-19T00:13:18,540 INFO [master/6b1531871d35:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-19T00:13:18,542 INFO [master/6b1531871d35:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-19T00:13:18,543 INFO [master/6b1531871d35:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-19T00:13:18,543 INFO [master/6b1531871d35:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-19T00:13:18,546 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6b1531871d35,33843,1731975193570-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T00:13:18,547 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6b1531871d35,33843,1731975193570-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-19T00:13:18,566 DEBUG [master/6b1531871d35:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-19T00:13:18,567 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-19T00:13:18,568 INFO [master/6b1531871d35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6b1531871d35,33843,1731975193570-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T00:13:18,630 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:18,633 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-19T00:13:18,633 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-19T00:13:18,636 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:13:18,639 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:13:18,649 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:13:18,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:13:18,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:13:18,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b5c7a62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:18,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:13:18,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:13:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:18,664 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:13:18,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:18,669 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:13:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:13:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:13:18,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:13:18,709 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=6b1531871d35,33843,1731975193570 2024-11-19T00:13:18,724 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=231, ProcessCount=11, AvailableMemoryMB=2432 2024-11-19T00:13:18,735 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T00:13:18,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 6b1531871d35,33843,1731975193570 2024-11-19T00:13:18,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@35874ff9 2024-11-19T00:13:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T00:13:18,798 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T00:13:18,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T00:13:18,816 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T00:13:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T00:13:18,832 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T00:13:18,835 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:18,838 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-19T00:13:18,839 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T00:13:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T00:13:18,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741835_1011 (size=1061) 2024-11-19T00:13:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T00:13:19,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T00:13:19,264 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2 2024-11-19T00:13:19,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741836_1012 (size=53) 2024-11-19T00:13:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T00:13:19,675 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:13:19,675 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing df30878ae408737da14ed71330503a0a, disabling compactions & flushes 2024-11-19T00:13:19,676 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:19,676 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:19,676 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. after waiting 0 ms 2024-11-19T00:13:19,676 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:19,676 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:19,676 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for df30878ae408737da14ed71330503a0a: Waiting for close lock at 1731975199675Disabling compacts and flushes for region at 1731975199675Disabling writes for close at 1731975199676 (+1 ms)Writing region close event to WAL at 1731975199676Closed at 1731975199676 2024-11-19T00:13:19,679 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T00:13:19,684 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731975199679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731975199679"}]},"ts":"1731975199679"} 2024-11-19T00:13:19,690 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T00:13:19,693 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T00:13:19,697 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975199693"}]},"ts":"1731975199693"} 2024-11-19T00:13:19,703 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T00:13:19,706 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=df30878ae408737da14ed71330503a0a, ASSIGN}] 2024-11-19T00:13:19,709 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=df30878ae408737da14ed71330503a0a, ASSIGN 2024-11-19T00:13:19,712 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=df30878ae408737da14ed71330503a0a, ASSIGN; state=OFFLINE, location=6b1531871d35,38567,1731975194317; forceNewPlan=false, retain=false 2024-11-19T00:13:19,863 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=df30878ae408737da14ed71330503a0a, regionState=OPENING, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:13:19,868 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=df30878ae408737da14ed71330503a0a, ASSIGN because future has completed 2024-11-19T00:13:19,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317}] 2024-11-19T00:13:19,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T00:13:20,036 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:20,036 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} 2024-11-19T00:13:20,037 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees df30878ae408737da14ed71330503a0a 2024-11-19T00:13:20,037 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:13:20,037 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for df30878ae408737da14ed71330503a0a 2024-11-19T00:13:20,038 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for df30878ae408737da14ed71330503a0a 2024-11-19T00:13:20,041 INFO [StoreOpener-df30878ae408737da14ed71330503a0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region df30878ae408737da14ed71330503a0a 2024-11-19T00:13:20,046 INFO [StoreOpener-df30878ae408737da14ed71330503a0a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:13:20,047 INFO [StoreOpener-df30878ae408737da14ed71330503a0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df30878ae408737da14ed71330503a0a columnFamilyName A 2024-11-19T00:13:20,047 DEBUG [StoreOpener-df30878ae408737da14ed71330503a0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:20,049 INFO [StoreOpener-df30878ae408737da14ed71330503a0a-1 {}] regionserver.HStore(327): Store=df30878ae408737da14ed71330503a0a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:13:20,050 INFO [StoreOpener-df30878ae408737da14ed71330503a0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region df30878ae408737da14ed71330503a0a 2024-11-19T00:13:20,052 INFO [StoreOpener-df30878ae408737da14ed71330503a0a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:13:20,053 INFO [StoreOpener-df30878ae408737da14ed71330503a0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df30878ae408737da14ed71330503a0a columnFamilyName B 2024-11-19T00:13:20,053 DEBUG [StoreOpener-df30878ae408737da14ed71330503a0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:20,054 INFO [StoreOpener-df30878ae408737da14ed71330503a0a-1 {}] regionserver.HStore(327): Store=df30878ae408737da14ed71330503a0a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:13:20,054 INFO [StoreOpener-df30878ae408737da14ed71330503a0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region df30878ae408737da14ed71330503a0a 2024-11-19T00:13:20,056 INFO [StoreOpener-df30878ae408737da14ed71330503a0a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:13:20,057 INFO [StoreOpener-df30878ae408737da14ed71330503a0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df30878ae408737da14ed71330503a0a columnFamilyName C 2024-11-19T00:13:20,057 DEBUG [StoreOpener-df30878ae408737da14ed71330503a0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:20,058 INFO [StoreOpener-df30878ae408737da14ed71330503a0a-1 {}] regionserver.HStore(327): Store=df30878ae408737da14ed71330503a0a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:13:20,058 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:20,059 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for df30878ae408737da14ed71330503a0a 2024-11-19T00:13:20,060 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a 2024-11-19T00:13:20,061 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a 2024-11-19T00:13:20,063 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for df30878ae408737da14ed71330503a0a 2024-11-19T00:13:20,063 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for df30878ae408737da14ed71330503a0a 2024-11-19T00:13:20,065 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T00:13:20,067 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for df30878ae408737da14ed71330503a0a 2024-11-19T00:13:20,071 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T00:13:20,073 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened df30878ae408737da14ed71330503a0a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68396521, jitterRate=0.01918758451938629}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T00:13:20,073 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for df30878ae408737da14ed71330503a0a 2024-11-19T00:13:20,074 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for df30878ae408737da14ed71330503a0a: Running coprocessor pre-open hook at 1731975200038Writing region info on filesystem at 1731975200038Initializing all the Stores at 1731975200040 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975200040Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975200040Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975200040Cleaning up temporary data from old regions at 1731975200063 (+23 ms)Running coprocessor post-open hooks at 1731975200073 (+10 ms)Region opened successfully at 1731975200074 (+1 ms) 2024-11-19T00:13:20,077 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., pid=6, masterSystemTime=1731975200024 2024-11-19T00:13:20,081 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:20,081 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:20,083 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=df30878ae408737da14ed71330503a0a, regionState=OPEN, openSeqNum=2, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:13:20,087 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 because future has completed 2024-11-19T00:13:20,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-19T00:13:20,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 in 220 msec 2024-11-19T00:13:20,100 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-19T00:13:20,100 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=df30878ae408737da14ed71330503a0a, ASSIGN in 388 msec 2024-11-19T00:13:20,101 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T00:13:20,102 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975200102"}]},"ts":"1731975200102"} 2024-11-19T00:13:20,105 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T00:13:20,107 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T00:13:20,111 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2870 sec 2024-11-19T00:13:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T00:13:21,005 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:13:21,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,010 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:13:21,011 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:13:21,013 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:13:21,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:13:21,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:13:21,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7983b71f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:13:21,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:13:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:21,019 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:13:21,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e34b7e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,022 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:13:21,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:13:21,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:13:21,028 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:13:21,031 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T00:13:21,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 6b1531871d35,33843,1731975193570 2024-11-19T00:13:21,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2d5e92ab 2024-11-19T00:13:21,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T00:13:21,038 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T00:13:21,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,044 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:13:21,045 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:13:21,046 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:13:21,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:13:21,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:13:21,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ffd0298, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:13:21,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:13:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:21,051 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:13:21,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30bed14c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,054 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,054 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:13:21,054 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:13:21,056 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:13:21,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:13:21,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:13:21,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b130fb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:13:21,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:13:21,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:21,062 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:13:21,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1d7645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,065 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:13:21,065 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:13:21,069 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:13:21,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:13:21,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:13:21,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1612488f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:13:21,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:13:21,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:21,075 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:13:21,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff1f1f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,078 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,078 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:13:21,078 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:13:21,083 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:13:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:13:21,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:13:21,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ec01134, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:13:21,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:13:21,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:21,089 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:13:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a081598, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,092 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,092 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:13:21,092 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:13:21,093 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:13:21,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:13:21,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:13:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5805ae29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:13:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:13:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:21,097 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:13:21,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d230d79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,100 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,100 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:13:21,100 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:13:21,101 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:13:21,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:13:21,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:13:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29adf99b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:13:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:13:21,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:21,105 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:13:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d14565c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,110 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:13:21,110 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:13:21,111 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:13:21,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:13:21,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:13:21,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d22a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:13:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:13:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:21,114 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:13:21,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@545d818d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,117 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,117 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:13:21,117 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:13:21,118 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:13:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:13:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:13:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c710cac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:13:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:13:21,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:21,122 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:13:21,123 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f6fe38c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,130 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,130 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:13:21,131 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:13:21,132 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:13:21,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:13:21,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:13:21,133 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d56454e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,133 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:13:21,133 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:13:21,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:21,135 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:13:21,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6514b0c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:21,141 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:13:21,141 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:13:21,141 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:13:21,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:13:21,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:13:21,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:13:21,143 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:13:21,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:13:21,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:13:21,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:13:21,145 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:13:21,145 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:13:21,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:13:21,145 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:13:21,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:13:21,146 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:13:21,146 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:13:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:13:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:13:21,148 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:13:21,148 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:13:21,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:13:21,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:13:21,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:13:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:13:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:13:21,154 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:13:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:13:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:13:21,154 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:13:21,155 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:13:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:13:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:13:21,158 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:13:21,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:13:21,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:13:21,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:13:21,162 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:13:21,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:13:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:13:21,168 DEBUG [Thread-156 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:13:21,170 DEBUG [Thread-156 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:13:21,172 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:13:21,177 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:13:21,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:13:21,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:13:21,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-19T00:13:21,193 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:13:21,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T00:13:21,196 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:13:21,198 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:13:21,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on df30878ae408737da14ed71330503a0a 2024-11-19T00:13:21,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T00:13:21,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:21,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:21,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:21,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:21,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:21,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:21,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T00:13:21,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T00:13:21,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:21,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:21,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:21,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:21,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:21,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:21,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975261367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975261367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:21,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975261373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:21,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:21,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975261373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:21,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:21,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:21,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:21,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:21,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975261386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:21,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:21,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/c90ea3b073f844e6b908f37f5b3a3db8 is 50, key is test_row_0/A:col10/1731975201164/Put/seqid=0 2024-11-19T00:13:21,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741837_1013 (size=12001) 2024-11-19T00:13:21,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/c90ea3b073f844e6b908f37f5b3a3db8 2024-11-19T00:13:21,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:21,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975261506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:21,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:21,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:21,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975261507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:21,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975261507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:21,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:21,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:21,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:21,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975261508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:21,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T00:13:21,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:21,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975261508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:21,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:21,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T00:13:21,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:21,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:21,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:21,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:21,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/d289013c3d55418aae749e7e77a4a6f8 is 50, key is test_row_0/B:col10/1731975201164/Put/seqid=0 2024-11-19T00:13:21,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:21,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741838_1014 (size=12001) 2024-11-19T00:13:21,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975261714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:21,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:21,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:21,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:21,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975261727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:21,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975261727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:21,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975261727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:21,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:21,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975261728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:21,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:21,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:21,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T00:13:21,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:21,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:21,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:21,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:21,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:21,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T00:13:21,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T00:13:21,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:21,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:21,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:21,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:21,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:22,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/d289013c3d55418aae749e7e77a4a6f8 2024-11-19T00:13:22,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975262025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:22,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:22,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:22,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:22,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975262045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:22,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975262045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:22,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975262047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:22,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975262047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:22,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:22,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:22,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:22,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:22,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:22,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:22,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:22,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:22,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:22,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T00:13:22,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:22,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:22,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:22,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:22,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:22,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/039566c1a4084b0d9f662f8a23a9ffbe is 50, key is test_row_0/C:col10/1731975201164/Put/seqid=0 2024-11-19T00:13:22,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741839_1015 (size=12001) 2024-11-19T00:13:22,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T00:13:22,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:22,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:22,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:22,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:22,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T00:13:22,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T00:13:22,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:22,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:22,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:22,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:22,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:22,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975262535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:22,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T00:13:22,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:22,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:22,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:22,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:22,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:22,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/039566c1a4084b0d9f662f8a23a9ffbe 2024-11-19T00:13:22,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/c90ea3b073f844e6b908f37f5b3a3db8 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/c90ea3b073f844e6b908f37f5b3a3db8 2024-11-19T00:13:22,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975262565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:22,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:22,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:22,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:22,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975262565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:22,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:22,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:22,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:22,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:22,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975262575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975262575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:22,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:22,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:22,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:22,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:22,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:22,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:22,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/c90ea3b073f844e6b908f37f5b3a3db8, entries=150, sequenceid=13, filesize=11.7 K 2024-11-19T00:13:22,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/d289013c3d55418aae749e7e77a4a6f8 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/d289013c3d55418aae749e7e77a4a6f8 2024-11-19T00:13:22,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/d289013c3d55418aae749e7e77a4a6f8, entries=150, sequenceid=13, filesize=11.7 K 2024-11-19T00:13:22,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/039566c1a4084b0d9f662f8a23a9ffbe as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/039566c1a4084b0d9f662f8a23a9ffbe 2024-11-19T00:13:22,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/039566c1a4084b0d9f662f8a23a9ffbe, entries=150, sequenceid=13, filesize=11.7 K 2024-11-19T00:13:22,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for df30878ae408737da14ed71330503a0a in 1400ms, sequenceid=13, compaction requested=false 2024-11-19T00:13:22,658 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T00:13:22,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:22,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T00:13:22,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:22,699 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T00:13:22,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:22,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:22,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:22,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:22,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:22,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:22,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/66fc606e515e47a9a071849e405f9aea is 50, key is test_row_0/A:col10/1731975201380/Put/seqid=0 2024-11-19T00:13:22,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741840_1016 (size=12001) 2024-11-19T00:13:22,755 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/66fc606e515e47a9a071849e405f9aea 2024-11-19T00:13:22,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/8ad2725a53f74e17ad6df02f587ab850 is 50, key is test_row_0/B:col10/1731975201380/Put/seqid=0 2024-11-19T00:13:22,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741841_1017 (size=12001) 2024-11-19T00:13:22,832 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/8ad2725a53f74e17ad6df02f587ab850 2024-11-19T00:13:22,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/3b4f3e8c43a946329842ddd3030b79c0 is 50, key is test_row_0/C:col10/1731975201380/Put/seqid=0 2024-11-19T00:13:22,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741842_1018 (size=12001) 2024-11-19T00:13:22,877 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/3b4f3e8c43a946329842ddd3030b79c0 2024-11-19T00:13:22,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/66fc606e515e47a9a071849e405f9aea as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/66fc606e515e47a9a071849e405f9aea 2024-11-19T00:13:22,904 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-19T00:13:22,921 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/66fc606e515e47a9a071849e405f9aea, entries=150, sequenceid=33, filesize=11.7 K 2024-11-19T00:13:22,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/8ad2725a53f74e17ad6df02f587ab850 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/8ad2725a53f74e17ad6df02f587ab850 2024-11-19T00:13:22,950 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/8ad2725a53f74e17ad6df02f587ab850, entries=150, sequenceid=33, filesize=11.7 K 2024-11-19T00:13:22,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/3b4f3e8c43a946329842ddd3030b79c0 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/3b4f3e8c43a946329842ddd3030b79c0 2024-11-19T00:13:22,971 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/3b4f3e8c43a946329842ddd3030b79c0, entries=150, sequenceid=33, filesize=11.7 K 2024-11-19T00:13:22,974 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for df30878ae408737da14ed71330503a0a in 275ms, sequenceid=33, compaction requested=false 2024-11-19T00:13:22,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:22,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:22,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-19T00:13:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-19T00:13:22,992 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-19T00:13:22,992 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7850 sec 2024-11-19T00:13:22,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.8150 sec 2024-11-19T00:13:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T00:13:23,344 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:13:23,345 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:13:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-19T00:13:23,350 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:13:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T00:13:23,352 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:13:23,353 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:13:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T00:13:23,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T00:13:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-19T00:13:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-19T00:13:23,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-19T00:13:23,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 164 msec 2024-11-19T00:13:23,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 178 msec 2024-11-19T00:13:23,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T00:13:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on df30878ae408737da14ed71330503a0a 2024-11-19T00:13:23,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:23,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:23,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:23,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:23,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:23,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:23,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/12d77166d314435fa88359e10fdd1f0c is 50, key is test_row_0/A:col10/1731975203596/Put/seqid=0 2024-11-19T00:13:23,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741843_1019 (size=16681) 2024-11-19T00:13:23,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/12d77166d314435fa88359e10fdd1f0c 2024-11-19T00:13:23,656 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T00:13:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T00:13:23,665 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:13:23,665 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:13:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-19T00:13:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T00:13:23,671 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:13:23,674 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:13:23,674 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:13:23,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/e62d87fd768a44efb70074b77b86338f is 50, key is test_row_0/B:col10/1731975203596/Put/seqid=0 2024-11-19T00:13:23,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:23,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975263662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:23,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:23,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975263663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:23,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975263673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:23,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:23,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:23,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:23,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975263677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:23,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975263680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:23,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:23,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741844_1020 (size=12001) 2024-11-19T00:13:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T00:13:23,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975263784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:23,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:23,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:23,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:23,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975263796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:23,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975263796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:23,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975263796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:23,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:23,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:23,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975263797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:23,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:23,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:23,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:23,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:23,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:23,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:23,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:23,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T00:13:23,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:23,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:23,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:23,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:23,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:23,923 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-19T00:13:23,923 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-19T00:13:23,926 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T00:13:23,926 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-19T00:13:23,926 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-19T00:13:23,927 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-19T00:13:23,928 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T00:13:23,928 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T00:13:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T00:13:23,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T00:13:23,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:23,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:23,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:23,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:23,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:23,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975263995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:24,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975264004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:24,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975264014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:24,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975264015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:24,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975264015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:24,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/e62d87fd768a44efb70074b77b86338f 2024-11-19T00:13:24,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T00:13:24,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:24,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:24,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:24,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:24,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:24,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:24,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/506912e6030546f5a80dd4b1e50366e3 is 50, key is test_row_0/C:col10/1731975203596/Put/seqid=0 2024-11-19T00:13:24,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741845_1021 (size=12001) 2024-11-19T00:13:24,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/506912e6030546f5a80dd4b1e50366e3 2024-11-19T00:13:24,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/12d77166d314435fa88359e10fdd1f0c as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/12d77166d314435fa88359e10fdd1f0c 2024-11-19T00:13:24,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/12d77166d314435fa88359e10fdd1f0c, entries=250, sequenceid=46, filesize=16.3 K 2024-11-19T00:13:24,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/e62d87fd768a44efb70074b77b86338f as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/e62d87fd768a44efb70074b77b86338f 2024-11-19T00:13:24,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/e62d87fd768a44efb70074b77b86338f, entries=150, sequenceid=46, filesize=11.7 K 2024-11-19T00:13:24,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/506912e6030546f5a80dd4b1e50366e3 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/506912e6030546f5a80dd4b1e50366e3 2024-11-19T00:13:24,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/506912e6030546f5a80dd4b1e50366e3, entries=150, sequenceid=46, filesize=11.7 K 2024-11-19T00:13:24,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for df30878ae408737da14ed71330503a0a in 681ms, sequenceid=46, compaction requested=true 2024-11-19T00:13:24,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T00:13:24,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T00:13:24,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:24,300 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T00:13:24,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:24,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:24,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:24,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:24,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:24,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:24,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:13:24,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:24,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:13:24,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:24,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:13:24,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:24,312 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:24,315 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:24,317 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on df30878ae408737da14ed71330503a0a 2024-11-19T00:13:24,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:24,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/6d47bb477710409a891bb44cf3c68187 is 50, key is test_row_0/A:col10/1731975203670/Put/seqid=0 2024-11-19T00:13:24,326 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/B is initiating minor compaction (all files) 2024-11-19T00:13:24,327 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/B in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:24,327 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:24,327 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/d289013c3d55418aae749e7e77a4a6f8, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/8ad2725a53f74e17ad6df02f587ab850, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/e62d87fd768a44efb70074b77b86338f] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=35.2 K 2024-11-19T00:13:24,327 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/A is initiating minor compaction (all files) 2024-11-19T00:13:24,329 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting d289013c3d55418aae749e7e77a4a6f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731975201164 2024-11-19T00:13:24,329 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/A in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:24,330 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/c90ea3b073f844e6b908f37f5b3a3db8, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/66fc606e515e47a9a071849e405f9aea, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/12d77166d314435fa88359e10fdd1f0c] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=39.7 K 2024-11-19T00:13:24,335 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting c90ea3b073f844e6b908f37f5b3a3db8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731975201164 2024-11-19T00:13:24,335 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ad2725a53f74e17ad6df02f587ab850, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731975201354 2024-11-19T00:13:24,337 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting e62d87fd768a44efb70074b77b86338f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731975203596 2024-11-19T00:13:24,337 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66fc606e515e47a9a071849e405f9aea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731975201354 2024-11-19T00:13:24,340 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12d77166d314435fa88359e10fdd1f0c, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731975203579 2024-11-19T00:13:24,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741846_1022 (size=12001) 2024-11-19T00:13:24,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:24,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975264345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:24,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:24,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:24,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975264345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:24,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:24,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:24,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975264348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:24,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:24,387 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#B#compaction#10 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:24,388 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/cb3f022673424792ab1d29199dca6463 is 50, key is test_row_0/B:col10/1731975203596/Put/seqid=0 2024-11-19T00:13:24,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:24,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975264374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975264378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:24,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:24,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:24,406 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#A#compaction#11 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:24,407 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/f125f13092f446e0a12ef0cf43648cfb is 50, key is test_row_0/A:col10/1731975203596/Put/seqid=0 2024-11-19T00:13:24,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741847_1023 (size=12104) 2024-11-19T00:13:24,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741848_1024 (size=12104) 2024-11-19T00:13:24,474 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/f125f13092f446e0a12ef0cf43648cfb as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/f125f13092f446e0a12ef0cf43648cfb 2024-11-19T00:13:24,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975264484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:24,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:24,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975264484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:24,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975264485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:24,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:24,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975264505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975264505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:24,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:24,516 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df30878ae408737da14ed71330503a0a/A of df30878ae408737da14ed71330503a0a into f125f13092f446e0a12ef0cf43648cfb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:24,516 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:24,520 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/A, priority=13, startTime=1731975204286; duration=0sec 2024-11-19T00:13:24,520 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:24,521 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:A 2024-11-19T00:13:24,521 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:24,524 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:24,524 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/C is initiating minor compaction (all files) 2024-11-19T00:13:24,524 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/C in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:24,526 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/039566c1a4084b0d9f662f8a23a9ffbe, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/3b4f3e8c43a946329842ddd3030b79c0, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/506912e6030546f5a80dd4b1e50366e3] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=35.2 K 2024-11-19T00:13:24,528 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 039566c1a4084b0d9f662f8a23a9ffbe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731975201164 2024-11-19T00:13:24,530 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b4f3e8c43a946329842ddd3030b79c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731975201354 2024-11-19T00:13:24,533 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 506912e6030546f5a80dd4b1e50366e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731975203596 2024-11-19T00:13:24,566 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#C#compaction#12 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:24,567 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/b77013f572824aff84678aa02670dfe4 is 50, key is test_row_0/C:col10/1731975203596/Put/seqid=0 2024-11-19T00:13:24,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741849_1025 (size=12104) 2024-11-19T00:13:24,626 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/b77013f572824aff84678aa02670dfe4 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/b77013f572824aff84678aa02670dfe4 2024-11-19T00:13:24,643 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df30878ae408737da14ed71330503a0a/C of df30878ae408737da14ed71330503a0a into b77013f572824aff84678aa02670dfe4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:24,643 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:24,643 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/C, priority=13, startTime=1731975204309; duration=0sec 2024-11-19T00:13:24,644 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:24,644 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:C 2024-11-19T00:13:24,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:24,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975264695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975264695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:24,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:24,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975264695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:24,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975264714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:24,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:24,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975264714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:24,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:24,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:24,779 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/6d47bb477710409a891bb44cf3c68187 2024-11-19T00:13:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T00:13:24,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/673e9b899b3e4a1e8665ad26f2e25adb is 50, key is test_row_0/B:col10/1731975203670/Put/seqid=0 2024-11-19T00:13:24,840 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/cb3f022673424792ab1d29199dca6463 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/cb3f022673424792ab1d29199dca6463 2024-11-19T00:13:24,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741850_1026 (size=12001) 2024-11-19T00:13:24,861 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df30878ae408737da14ed71330503a0a/B of df30878ae408737da14ed71330503a0a into cb3f022673424792ab1d29199dca6463(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:24,861 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:24,861 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/B, priority=13, startTime=1731975204309; duration=0sec 2024-11-19T00:13:24,861 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:24,861 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:B 2024-11-19T00:13:24,862 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/673e9b899b3e4a1e8665ad26f2e25adb 2024-11-19T00:13:24,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/5d3856aa7a644f6ea4459d1a7b798030 is 50, key is test_row_0/C:col10/1731975203670/Put/seqid=0 2024-11-19T00:13:24,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741851_1027 (size=12001) 2024-11-19T00:13:24,929 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/5d3856aa7a644f6ea4459d1a7b798030 2024-11-19T00:13:24,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/6d47bb477710409a891bb44cf3c68187 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/6d47bb477710409a891bb44cf3c68187 2024-11-19T00:13:24,962 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/6d47bb477710409a891bb44cf3c68187, entries=150, sequenceid=66, filesize=11.7 K 2024-11-19T00:13:24,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/673e9b899b3e4a1e8665ad26f2e25adb as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/673e9b899b3e4a1e8665ad26f2e25adb 2024-11-19T00:13:24,980 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/673e9b899b3e4a1e8665ad26f2e25adb, entries=150, sequenceid=66, filesize=11.7 K 2024-11-19T00:13:24,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/5d3856aa7a644f6ea4459d1a7b798030 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/5d3856aa7a644f6ea4459d1a7b798030 2024-11-19T00:13:24,999 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/5d3856aa7a644f6ea4459d1a7b798030, entries=150, sequenceid=66, filesize=11.7 K 2024-11-19T00:13:25,000 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for df30878ae408737da14ed71330503a0a in 701ms, sequenceid=66, compaction requested=false 2024-11-19T00:13:25,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:25,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:25,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-19T00:13:25,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-19T00:13:25,011 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-19T00:13:25,011 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3290 sec 2024-11-19T00:13:25,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on df30878ae408737da14ed71330503a0a 2024-11-19T00:13:25,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T00:13:25,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.3480 sec 2024-11-19T00:13:25,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:25,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:25,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:25,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:25,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:25,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:25,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/c2a7411acb1b453ab469b5038d558b5c is 50, key is test_row_0/A:col10/1731975205013/Put/seqid=0 2024-11-19T00:13:25,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741852_1028 (size=16681) 2024-11-19T00:13:25,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/c2a7411acb1b453ab469b5038d558b5c 2024-11-19T00:13:25,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/5e8fd30446354b3da5b89dd6d501ee66 is 50, key is test_row_0/B:col10/1731975205013/Put/seqid=0 2024-11-19T00:13:25,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741853_1029 (size=12001) 2024-11-19T00:13:25,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975265125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975265127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975265136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975265136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975265137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975265244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975265245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975265255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975265255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975265255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975265454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975265454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975265465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975265466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975265466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/5e8fd30446354b3da5b89dd6d501ee66 2024-11-19T00:13:25,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/522f73e8f23a483a8b207fb0651be6c0 is 50, key is test_row_0/C:col10/1731975205013/Put/seqid=0 2024-11-19T00:13:25,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741854_1030 (size=12001) 2024-11-19T00:13:25,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/522f73e8f23a483a8b207fb0651be6c0 2024-11-19T00:13:25,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/c2a7411acb1b453ab469b5038d558b5c as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/c2a7411acb1b453ab469b5038d558b5c 2024-11-19T00:13:25,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/c2a7411acb1b453ab469b5038d558b5c, entries=250, sequenceid=83, filesize=16.3 K 2024-11-19T00:13:25,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/5e8fd30446354b3da5b89dd6d501ee66 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/5e8fd30446354b3da5b89dd6d501ee66 2024-11-19T00:13:25,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/5e8fd30446354b3da5b89dd6d501ee66, entries=150, sequenceid=83, filesize=11.7 K 2024-11-19T00:13:25,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/522f73e8f23a483a8b207fb0651be6c0 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/522f73e8f23a483a8b207fb0651be6c0 2024-11-19T00:13:25,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/522f73e8f23a483a8b207fb0651be6c0, entries=150, sequenceid=83, filesize=11.7 K 2024-11-19T00:13:25,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for df30878ae408737da14ed71330503a0a in 616ms, sequenceid=83, compaction requested=true 2024-11-19T00:13:25,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:25,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:13:25,632 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:25,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:25,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:13:25,632 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:25,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:25,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:13:25,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:25,634 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:25,635 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/A is initiating minor compaction (all files) 2024-11-19T00:13:25,635 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/A in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:25,635 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/f125f13092f446e0a12ef0cf43648cfb, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/6d47bb477710409a891bb44cf3c68187, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/c2a7411acb1b453ab469b5038d558b5c] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=39.8 K 2024-11-19T00:13:25,637 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:25,637 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/B is initiating minor compaction (all files) 2024-11-19T00:13:25,637 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/B in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:25,637 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/cb3f022673424792ab1d29199dca6463, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/673e9b899b3e4a1e8665ad26f2e25adb, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/5e8fd30446354b3da5b89dd6d501ee66] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=35.3 K 2024-11-19T00:13:25,638 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting f125f13092f446e0a12ef0cf43648cfb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731975203596 2024-11-19T00:13:25,639 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting cb3f022673424792ab1d29199dca6463, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731975203596 2024-11-19T00:13:25,639 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d47bb477710409a891bb44cf3c68187, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731975203659 2024-11-19T00:13:25,639 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 673e9b899b3e4a1e8665ad26f2e25adb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731975203659 2024-11-19T00:13:25,640 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2a7411acb1b453ab469b5038d558b5c, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731975204334 2024-11-19T00:13:25,640 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e8fd30446354b3da5b89dd6d501ee66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731975204369 2024-11-19T00:13:25,661 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#B#compaction#18 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:25,662 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/16f4a9668cc24f7d86dafd8ceab20c03 is 50, key is test_row_0/B:col10/1731975205013/Put/seqid=0 2024-11-19T00:13:25,663 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:25,664 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/c4cb4f16b92d4c08a1e5ed512860db1c is 50, key is test_row_0/A:col10/1731975205013/Put/seqid=0 2024-11-19T00:13:25,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741855_1031 (size=12207) 2024-11-19T00:13:25,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741856_1032 (size=12207) 2024-11-19T00:13:25,720 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/16f4a9668cc24f7d86dafd8ceab20c03 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/16f4a9668cc24f7d86dafd8ceab20c03 2024-11-19T00:13:25,737 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df30878ae408737da14ed71330503a0a/B of df30878ae408737da14ed71330503a0a into 16f4a9668cc24f7d86dafd8ceab20c03(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:25,737 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:25,737 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/B, priority=13, startTime=1731975205632; duration=0sec 2024-11-19T00:13:25,738 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:25,738 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:B 2024-11-19T00:13:25,738 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:25,741 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:25,741 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/C is initiating minor compaction (all files) 2024-11-19T00:13:25,741 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/C in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:25,741 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/b77013f572824aff84678aa02670dfe4, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/5d3856aa7a644f6ea4459d1a7b798030, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/522f73e8f23a483a8b207fb0651be6c0] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=35.3 K 2024-11-19T00:13:25,746 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting b77013f572824aff84678aa02670dfe4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731975203596 2024-11-19T00:13:25,750 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d3856aa7a644f6ea4459d1a7b798030, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731975203659 2024-11-19T00:13:25,752 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 522f73e8f23a483a8b207fb0651be6c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731975204369 2024-11-19T00:13:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on df30878ae408737da14ed71330503a0a 2024-11-19T00:13:25,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T00:13:25,786 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#C#compaction#20 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:25,786 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/a048092119804410a5398c1f24e02551 is 50, key is test_row_0/C:col10/1731975205013/Put/seqid=0 2024-11-19T00:13:25,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:25,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:25,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:25,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:25,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:25,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:25,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741857_1033 (size=12207) 2024-11-19T00:13:25,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/cc26c3f26fac422089c0e09382cc8c07 is 50, key is test_row_0/A:col10/1731975205782/Put/seqid=0 2024-11-19T00:13:25,812 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/a048092119804410a5398c1f24e02551 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/a048092119804410a5398c1f24e02551 2024-11-19T00:13:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T00:13:25,815 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:13:25,816 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:13:25,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-19T00:13:25,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T00:13:25,820 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:13:25,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975265809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975265811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,823 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:13:25,823 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:13:25,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975265815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975265815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975265820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,831 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df30878ae408737da14ed71330503a0a/C of df30878ae408737da14ed71330503a0a into a048092119804410a5398c1f24e02551(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:25,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,831 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:25,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,831 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/C, priority=13, startTime=1731975205632; duration=0sec 2024-11-19T00:13:25,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,832 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:25,832 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:C 2024-11-19T00:13:25,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741858_1034 (size=12001) 2024-11-19T00:13:25,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/cc26c3f26fac422089c0e09382cc8c07 2024-11-19T00:13:25,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/c1444bc6675640a7be0c8c3a92670828 is 50, key is test_row_0/B:col10/1731975205782/Put/seqid=0 2024-11-19T00:13:25,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741859_1035 (size=12001) 2024-11-19T00:13:25,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/c1444bc6675640a7be0c8c3a92670828 2024-11-19T00:13:25,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/771ef694a677497da4b0730f938730a5 is 50, key is test_row_0/C:col10/1731975205782/Put/seqid=0 2024-11-19T00:13:25,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741860_1036 (size=12001) 2024-11-19T00:13:25,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/771ef694a677497da4b0730f938730a5 2024-11-19T00:13:25,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/cc26c3f26fac422089c0e09382cc8c07 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/cc26c3f26fac422089c0e09382cc8c07 2024-11-19T00:13:25,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T00:13:25,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975265925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975265934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975265934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975265934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/cc26c3f26fac422089c0e09382cc8c07, entries=150, sequenceid=105, filesize=11.7 K 2024-11-19T00:13:25,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:25,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975265934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:25,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:25,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:25,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/c1444bc6675640a7be0c8c3a92670828 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/c1444bc6675640a7be0c8c3a92670828 2024-11-19T00:13:25,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/c1444bc6675640a7be0c8c3a92670828, entries=150, sequenceid=105, filesize=11.7 K 2024-11-19T00:13:25,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/771ef694a677497da4b0730f938730a5 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/771ef694a677497da4b0730f938730a5 2024-11-19T00:13:25,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/771ef694a677497da4b0730f938730a5, entries=150, sequenceid=105, filesize=11.7 K 2024-11-19T00:13:25,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for df30878ae408737da14ed71330503a0a in 193ms, sequenceid=105, compaction requested=false 2024-11-19T00:13:25,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:25,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T00:13:25,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:25,978 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T00:13:25,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:25,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:25,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:25,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:25,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:25,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:25,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/597cad2e0c8c49d8a2c2093124250e25 is 50, key is test_row_0/A:col10/1731975205805/Put/seqid=0 2024-11-19T00:13:26,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741861_1037 (size=12001) 2024-11-19T00:13:26,007 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/597cad2e0c8c49d8a2c2093124250e25 2024-11-19T00:13:26,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/0ed291fa135440488e9170294cb7d760 is 50, key is test_row_0/B:col10/1731975205805/Put/seqid=0 2024-11-19T00:13:26,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741862_1038 (size=12001) 2024-11-19T00:13:26,047 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/0ed291fa135440488e9170294cb7d760 2024-11-19T00:13:26,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/526fd821e9de44eab94a6e17fd68fc72 is 50, key is test_row_0/C:col10/1731975205805/Put/seqid=0 2024-11-19T00:13:26,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741863_1039 (size=12001) 2024-11-19T00:13:26,101 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/526fd821e9de44eab94a6e17fd68fc72 2024-11-19T00:13:26,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/597cad2e0c8c49d8a2c2093124250e25 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/597cad2e0c8c49d8a2c2093124250e25 2024-11-19T00:13:26,121 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/c4cb4f16b92d4c08a1e5ed512860db1c as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/c4cb4f16b92d4c08a1e5ed512860db1c 2024-11-19T00:13:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T00:13:26,135 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df30878ae408737da14ed71330503a0a/A of df30878ae408737da14ed71330503a0a into c4cb4f16b92d4c08a1e5ed512860db1c(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:26,135 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:26,135 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/A, priority=13, startTime=1731975205632; duration=0sec 2024-11-19T00:13:26,136 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:26,136 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:A 2024-11-19T00:13:26,138 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/597cad2e0c8c49d8a2c2093124250e25, entries=150, sequenceid=117, filesize=11.7 K 2024-11-19T00:13:26,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/0ed291fa135440488e9170294cb7d760 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/0ed291fa135440488e9170294cb7d760 2024-11-19T00:13:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on df30878ae408737da14ed71330503a0a 2024-11-19T00:13:26,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:26,162 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/0ed291fa135440488e9170294cb7d760, entries=150, sequenceid=117, filesize=11.7 K 2024-11-19T00:13:26,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/526fd821e9de44eab94a6e17fd68fc72 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/526fd821e9de44eab94a6e17fd68fc72 2024-11-19T00:13:26,179 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/526fd821e9de44eab94a6e17fd68fc72, entries=150, sequenceid=117, filesize=11.7 K 2024-11-19T00:13:26,182 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=73.80 KB/75570 for df30878ae408737da14ed71330503a0a in 203ms, sequenceid=117, compaction requested=true 2024-11-19T00:13:26,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:26,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:26,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-19T00:13:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-19T00:13:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on df30878ae408737da14ed71330503a0a 2024-11-19T00:13:26,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T00:13:26,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:26,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:26,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:26,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:26,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:26,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:26,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-19T00:13:26,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 362 msec 2024-11-19T00:13:26,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/84c133b233774befa09b0e58f8f18437 is 50, key is test_row_0/A:col10/1731975206181/Put/seqid=0 2024-11-19T00:13:26,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 382 msec 2024-11-19T00:13:26,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975266220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:26,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975266223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:26,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:26,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975266226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:26,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741864_1040 (size=12151) 2024-11-19T00:13:26,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:26,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975266227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:26,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/84c133b233774befa09b0e58f8f18437 2024-11-19T00:13:26,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975266233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:26,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:26,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/b2203caf1fab43b797828b414c62fbe9 is 50, key is test_row_0/B:col10/1731975206181/Put/seqid=0 2024-11-19T00:13:26,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741865_1041 (size=12151) 2024-11-19T00:13:26,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975266334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:26,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:26,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975266344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:26,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975266344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:26,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:26,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975266345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:26,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:26,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975266353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:26,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T00:13:26,445 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:13:26,446 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:13:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-19T00:13:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T00:13:26,450 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:13:26,451 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:13:26,452 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:13:26,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975266544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:26,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T00:13:26,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975266554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:26,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975266555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:26,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:26,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975266564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:26,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975266569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:26,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:26,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:26,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T00:13:26,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:26,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:26,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:26,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:26,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:26,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/b2203caf1fab43b797828b414c62fbe9 2024-11-19T00:13:26,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/7421f66d2ec1498b89b5982469336fdb is 50, key is test_row_0/C:col10/1731975206181/Put/seqid=0 2024-11-19T00:13:26,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741866_1042 (size=12151) 2024-11-19T00:13:26,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T00:13:26,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:26,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:26,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:26,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:26,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T00:13:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:26,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975266854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:26,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975266874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:26,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:26,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975266876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:26,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:26,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975266884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:26,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:26,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975266885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:26,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:26,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:26,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T00:13:26,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:26,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:26,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:26,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:26,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:27,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T00:13:27,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:27,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:27,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:27,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:27,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T00:13:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:27,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/7421f66d2ec1498b89b5982469336fdb 2024-11-19T00:13:27,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/84c133b233774befa09b0e58f8f18437 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/84c133b233774befa09b0e58f8f18437 2024-11-19T00:13:27,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/84c133b233774befa09b0e58f8f18437, entries=150, sequenceid=134, filesize=11.9 K 2024-11-19T00:13:27,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/b2203caf1fab43b797828b414c62fbe9 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/b2203caf1fab43b797828b414c62fbe9 2024-11-19T00:13:27,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/b2203caf1fab43b797828b414c62fbe9, entries=150, sequenceid=134, filesize=11.9 K 2024-11-19T00:13:27,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/7421f66d2ec1498b89b5982469336fdb as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/7421f66d2ec1498b89b5982469336fdb 2024-11-19T00:13:27,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/7421f66d2ec1498b89b5982469336fdb, entries=150, sequenceid=134, filesize=11.9 K 2024-11-19T00:13:27,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for df30878ae408737da14ed71330503a0a in 1022ms, sequenceid=134, compaction requested=true 2024-11-19T00:13:27,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:27,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:13:27,208 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:13:27,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:27,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:13:27,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:27,208 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:13:27,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:13:27,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:27,211 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:13:27,212 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/A is initiating minor compaction (all files) 2024-11-19T00:13:27,212 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/A in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:27,212 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/c4cb4f16b92d4c08a1e5ed512860db1c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/cc26c3f26fac422089c0e09382cc8c07, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/597cad2e0c8c49d8a2c2093124250e25, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/84c133b233774befa09b0e58f8f18437] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=47.2 K 2024-11-19T00:13:27,213 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:13:27,213 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/B is initiating minor compaction (all files) 2024-11-19T00:13:27,214 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/B in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:27,214 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/16f4a9668cc24f7d86dafd8ceab20c03, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/c1444bc6675640a7be0c8c3a92670828, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/0ed291fa135440488e9170294cb7d760, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/b2203caf1fab43b797828b414c62fbe9] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=47.2 K 2024-11-19T00:13:27,215 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4cb4f16b92d4c08a1e5ed512860db1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731975204369 2024-11-19T00:13:27,222 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc26c3f26fac422089c0e09382cc8c07, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731975205133 2024-11-19T00:13:27,223 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 16f4a9668cc24f7d86dafd8ceab20c03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731975204369 2024-11-19T00:13:27,223 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 597cad2e0c8c49d8a2c2093124250e25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731975205805 2024-11-19T00:13:27,224 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84c133b233774befa09b0e58f8f18437, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731975206168 2024-11-19T00:13:27,225 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting c1444bc6675640a7be0c8c3a92670828, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731975205133 2024-11-19T00:13:27,226 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ed291fa135440488e9170294cb7d760, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731975205805 2024-11-19T00:13:27,227 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting b2203caf1fab43b797828b414c62fbe9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731975206168 2024-11-19T00:13:27,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T00:13:27,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:27,229 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T00:13:27,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:27,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:27,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:27,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:27,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:27,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:27,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/b7b7af5cef3b4a3e8f8cdeb980a06205 is 50, key is test_row_0/A:col10/1731975206224/Put/seqid=0 2024-11-19T00:13:27,262 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#B#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:27,263 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/4fde3ec3efcd4b118b23c9a83aa91952 is 50, key is test_row_0/B:col10/1731975206181/Put/seqid=0 2024-11-19T00:13:27,270 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#A#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:27,271 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/4941ac08ecf443baa822f87797129587 is 50, key is test_row_0/A:col10/1731975206181/Put/seqid=0 2024-11-19T00:13:27,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741867_1043 (size=12151) 2024-11-19T00:13:27,296 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/b7b7af5cef3b4a3e8f8cdeb980a06205 2024-11-19T00:13:27,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741868_1044 (size=12493) 2024-11-19T00:13:27,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741869_1045 (size=12493) 2024-11-19T00:13:27,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/6154f5d0b359421f8137517913f45847 is 50, key is test_row_0/B:col10/1731975206224/Put/seqid=0 2024-11-19T00:13:27,334 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/4941ac08ecf443baa822f87797129587 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/4941ac08ecf443baa822f87797129587 2024-11-19T00:13:27,349 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in df30878ae408737da14ed71330503a0a/A of df30878ae408737da14ed71330503a0a into 4941ac08ecf443baa822f87797129587(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:27,349 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:27,349 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/A, priority=12, startTime=1731975207208; duration=0sec 2024-11-19T00:13:27,350 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:27,350 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:A 2024-11-19T00:13:27,350 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:13:27,353 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:13:27,354 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/C is initiating minor compaction (all files) 2024-11-19T00:13:27,354 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/C in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:27,354 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/a048092119804410a5398c1f24e02551, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/771ef694a677497da4b0730f938730a5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/526fd821e9de44eab94a6e17fd68fc72, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/7421f66d2ec1498b89b5982469336fdb] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=47.2 K 2024-11-19T00:13:27,355 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting a048092119804410a5398c1f24e02551, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731975204369 2024-11-19T00:13:27,356 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 771ef694a677497da4b0730f938730a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731975205133 2024-11-19T00:13:27,356 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 526fd821e9de44eab94a6e17fd68fc72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731975205805 2024-11-19T00:13:27,360 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7421f66d2ec1498b89b5982469336fdb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731975206168 2024-11-19T00:13:27,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741870_1046 (size=12151) 2024-11-19T00:13:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on df30878ae408737da14ed71330503a0a 2024-11-19T00:13:27,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:27,392 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#C#compaction#34 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:27,394 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/7237c46aa0374f48bc244373ef268421 is 50, key is test_row_0/C:col10/1731975206181/Put/seqid=0 2024-11-19T00:13:27,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741871_1047 (size=12493) 2024-11-19T00:13:27,430 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/7237c46aa0374f48bc244373ef268421 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/7237c46aa0374f48bc244373ef268421 2024-11-19T00:13:27,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:27,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:27,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975267426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:27,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975267432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:27,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:27,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:27,446 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in df30878ae408737da14ed71330503a0a/C of df30878ae408737da14ed71330503a0a into 7237c46aa0374f48bc244373ef268421(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:27,446 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:27,446 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/C, priority=12, startTime=1731975207208; duration=0sec 2024-11-19T00:13:27,446 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:27,446 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:C 2024-11-19T00:13:27,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975267443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975267444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:27,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975267446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:27,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:27,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975267556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:27,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975267556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:27,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:27,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975267565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:27,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:27,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975267565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975267565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:27,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:27,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T00:13:27,718 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/4fde3ec3efcd4b118b23c9a83aa91952 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/4fde3ec3efcd4b118b23c9a83aa91952 2024-11-19T00:13:27,737 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in df30878ae408737da14ed71330503a0a/B of df30878ae408737da14ed71330503a0a into 4fde3ec3efcd4b118b23c9a83aa91952(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:27,738 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:27,738 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/B, priority=12, startTime=1731975207208; duration=0sec 2024-11-19T00:13:27,738 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:27,738 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:B 2024-11-19T00:13:27,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:27,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975267764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:27,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:27,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:27,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975267766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:27,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:27,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975267773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:27,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:27,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975267774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:27,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:27,778 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/6154f5d0b359421f8137517913f45847 2024-11-19T00:13:27,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975267774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:27,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:27,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:27,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/228b39eef60645c9a06b090418fae81a is 50, key is test_row_0/C:col10/1731975206224/Put/seqid=0 2024-11-19T00:13:27,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741872_1048 (size=12151) 2024-11-19T00:13:28,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975268074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:28,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:28,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975268084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:28,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:28,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975268084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:28,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:28,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975268085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:28,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:28,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975268085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:28,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:28,219 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/228b39eef60645c9a06b090418fae81a 2024-11-19T00:13:28,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/b7b7af5cef3b4a3e8f8cdeb980a06205 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/b7b7af5cef3b4a3e8f8cdeb980a06205 2024-11-19T00:13:28,246 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/b7b7af5cef3b4a3e8f8cdeb980a06205, entries=150, sequenceid=150, filesize=11.9 K 2024-11-19T00:13:28,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/6154f5d0b359421f8137517913f45847 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/6154f5d0b359421f8137517913f45847 2024-11-19T00:13:28,260 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/6154f5d0b359421f8137517913f45847, entries=150, sequenceid=150, filesize=11.9 K 2024-11-19T00:13:28,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/228b39eef60645c9a06b090418fae81a as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/228b39eef60645c9a06b090418fae81a 2024-11-19T00:13:28,270 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/228b39eef60645c9a06b090418fae81a, entries=150, sequenceid=150, filesize=11.9 K 2024-11-19T00:13:28,272 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=100.63 KB/103050 for df30878ae408737da14ed71330503a0a in 1043ms, sequenceid=150, compaction requested=false 2024-11-19T00:13:28,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:28,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:28,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-19T00:13:28,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-19T00:13:28,278 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-19T00:13:28,278 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8230 sec 2024-11-19T00:13:28,282 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.8330 sec 2024-11-19T00:13:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T00:13:28,598 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:13:28,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:13:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on df30878ae408737da14ed71330503a0a 2024-11-19T00:13:28,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T00:13:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-19T00:13:28,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:28,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:28,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:28,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:28,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:28,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:28,602 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:13:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T00:13:28,603 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:13:28,603 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:13:28,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/e2665cf6a37a42ebb537184ca27f7bd4 is 50, key is test_row_0/A:col10/1731975208597/Put/seqid=0 2024-11-19T00:13:28,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741873_1049 (size=16931) 2024-11-19T00:13:28,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/e2665cf6a37a42ebb537184ca27f7bd4 2024-11-19T00:13:28,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975268614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:28,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975268616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:28,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:28,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:28,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975268621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:28,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:28,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975268622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:28,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:28,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975268626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:28,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/9b2861fa1e254defa7138d70fd97c6bc is 50, key is test_row_0/B:col10/1731975208597/Put/seqid=0 2024-11-19T00:13:28,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741874_1050 (size=12151) 2024-11-19T00:13:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T00:13:28,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975268726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:28,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975268735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:28,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:28,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:28,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975268735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975268735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:28,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:28,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:28,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975268735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:28,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:28,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T00:13:28,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:28,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:28,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:28,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:28,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:28,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T00:13:28,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:28,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:28,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:28,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:28,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:28,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T00:13:28,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:28,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975268934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:28,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:28,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:28,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975268944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:28,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:28,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:28,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975268944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:28,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:28,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:28,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975268945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:28,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975268954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:29,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/9b2861fa1e254defa7138d70fd97c6bc 2024-11-19T00:13:29,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/2d3a367f406b4c6ea5ff040c16cc36ad is 50, key is test_row_0/C:col10/1731975208597/Put/seqid=0 2024-11-19T00:13:29,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T00:13:29,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:29,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:29,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:29,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:29,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:29,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741875_1051 (size=12151) 2024-11-19T00:13:29,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T00:13:29,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:29,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:29,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:29,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:29,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T00:13:29,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:29,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975269244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975269254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:29,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:29,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:29,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975269254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:29,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:29,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975269264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:29,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975269264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:29,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T00:13:29,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:29,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:29,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:29,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:29,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:29,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:29,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/2d3a367f406b4c6ea5ff040c16cc36ad 2024-11-19T00:13:29,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/e2665cf6a37a42ebb537184ca27f7bd4 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/e2665cf6a37a42ebb537184ca27f7bd4 2024-11-19T00:13:29,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/e2665cf6a37a42ebb537184ca27f7bd4, entries=250, sequenceid=173, filesize=16.5 K 2024-11-19T00:13:29,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/9b2861fa1e254defa7138d70fd97c6bc as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/9b2861fa1e254defa7138d70fd97c6bc 2024-11-19T00:13:29,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/9b2861fa1e254defa7138d70fd97c6bc, entries=150, sequenceid=173, filesize=11.9 K 2024-11-19T00:13:29,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/2d3a367f406b4c6ea5ff040c16cc36ad as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/2d3a367f406b4c6ea5ff040c16cc36ad 2024-11-19T00:13:29,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T00:13:29,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:29,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:29,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:29,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:29,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:29,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/2d3a367f406b4c6ea5ff040c16cc36ad, entries=150, sequenceid=173, filesize=11.9 K 2024-11-19T00:13:29,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for df30878ae408737da14ed71330503a0a in 953ms, sequenceid=173, compaction requested=true 2024-11-19T00:13:29,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:29,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:13:29,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:29,551 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:29,552 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:29,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:13:29,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:29,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:13:29,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:29,553 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:29,553 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/B is initiating minor compaction (all files) 2024-11-19T00:13:29,553 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/B in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:29,554 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/4fde3ec3efcd4b118b23c9a83aa91952, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/6154f5d0b359421f8137517913f45847, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/9b2861fa1e254defa7138d70fd97c6bc] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=35.9 K 2024-11-19T00:13:29,554 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fde3ec3efcd4b118b23c9a83aa91952, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731975206168 2024-11-19T00:13:29,555 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 6154f5d0b359421f8137517913f45847, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731975206222 2024-11-19T00:13:29,555 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:29,556 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/A is initiating minor compaction (all files) 2024-11-19T00:13:29,556 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/A in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:29,556 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/4941ac08ecf443baa822f87797129587, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/b7b7af5cef3b4a3e8f8cdeb980a06205, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/e2665cf6a37a42ebb537184ca27f7bd4] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=40.6 K 2024-11-19T00:13:29,556 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b2861fa1e254defa7138d70fd97c6bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731975207420 2024-11-19T00:13:29,557 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4941ac08ecf443baa822f87797129587, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731975206168 2024-11-19T00:13:29,559 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7b7af5cef3b4a3e8f8cdeb980a06205, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731975206222 2024-11-19T00:13:29,560 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2665cf6a37a42ebb537184ca27f7bd4, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731975207406 2024-11-19T00:13:29,579 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#B#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:29,580 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/5afbe921fddc40cab85e14aa88283251 is 50, key is test_row_0/B:col10/1731975208597/Put/seqid=0 2024-11-19T00:13:29,589 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#A#compaction#40 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:29,589 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/30b65ad1b841437a81b1547ec91c36c4 is 50, key is test_row_0/A:col10/1731975208597/Put/seqid=0 2024-11-19T00:13:29,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741876_1052 (size=12595) 2024-11-19T00:13:29,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741877_1053 (size=12595) 2024-11-19T00:13:29,641 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/30b65ad1b841437a81b1547ec91c36c4 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/30b65ad1b841437a81b1547ec91c36c4 2024-11-19T00:13:29,667 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df30878ae408737da14ed71330503a0a/A of df30878ae408737da14ed71330503a0a into 30b65ad1b841437a81b1547ec91c36c4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:29,667 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:29,667 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/A, priority=13, startTime=1731975209551; duration=0sec 2024-11-19T00:13:29,667 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:29,667 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:A 2024-11-19T00:13:29,667 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:29,669 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:29,669 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/C is initiating minor compaction (all files) 2024-11-19T00:13:29,670 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/C in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:29,671 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/7237c46aa0374f48bc244373ef268421, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/228b39eef60645c9a06b090418fae81a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/2d3a367f406b4c6ea5ff040c16cc36ad] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=35.9 K 2024-11-19T00:13:29,671 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7237c46aa0374f48bc244373ef268421, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731975206168 2024-11-19T00:13:29,672 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 228b39eef60645c9a06b090418fae81a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731975206222 2024-11-19T00:13:29,673 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d3a367f406b4c6ea5ff040c16cc36ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731975207420 2024-11-19T00:13:29,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T00:13:29,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:29,688 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T00:13:29,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:29,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:29,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:29,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:29,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:29,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:29,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/20648145c6f645fdb73d2819597562a0 is 50, key is test_row_0/A:col10/1731975208606/Put/seqid=0 2024-11-19T00:13:29,699 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#C#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:29,704 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/a83b5b44f6e5432791395af0de664f38 is 50, key is test_row_0/C:col10/1731975208597/Put/seqid=0 2024-11-19T00:13:29,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741878_1054 (size=12151) 2024-11-19T00:13:29,717 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/20648145c6f645fdb73d2819597562a0 2024-11-19T00:13:29,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741879_1055 (size=12595) 2024-11-19T00:13:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T00:13:29,763 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/a83b5b44f6e5432791395af0de664f38 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/a83b5b44f6e5432791395af0de664f38 2024-11-19T00:13:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on df30878ae408737da14ed71330503a0a 2024-11-19T00:13:29,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:29,777 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df30878ae408737da14ed71330503a0a/C of df30878ae408737da14ed71330503a0a into a83b5b44f6e5432791395af0de664f38(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:29,777 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:29,777 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/C, priority=13, startTime=1731975209552; duration=0sec 2024-11-19T00:13:29,778 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:29,778 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:C 2024-11-19T00:13:29,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/db31719e72fb4d18ac7ec0295612e6a6 is 50, key is test_row_0/B:col10/1731975208606/Put/seqid=0 2024-11-19T00:13:29,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741880_1056 (size=12151) 2024-11-19T00:13:29,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:29,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:29,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975269798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:29,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975269799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:29,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975269805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:29,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:29,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:29,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975269812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:29,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975269814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:29,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:29,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975269924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:29,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:29,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975269925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:29,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:29,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975269925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:29,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:29,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:29,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975269925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975269925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:29,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:29,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:29,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:30,040 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/5afbe921fddc40cab85e14aa88283251 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/5afbe921fddc40cab85e14aa88283251 2024-11-19T00:13:30,050 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df30878ae408737da14ed71330503a0a/B of df30878ae408737da14ed71330503a0a into 5afbe921fddc40cab85e14aa88283251(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:30,050 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:30,050 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/B, priority=13, startTime=1731975209552; duration=0sec 2024-11-19T00:13:30,051 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:30,051 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:B 2024-11-19T00:13:30,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975270134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:30,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975270134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:30,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:30,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975270134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:30,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:30,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975270144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975270144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:30,205 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/db31719e72fb4d18ac7ec0295612e6a6 2024-11-19T00:13:30,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/db56c89a2988472591b61c1d4b7a5a49 is 50, key is test_row_0/C:col10/1731975208606/Put/seqid=0 2024-11-19T00:13:30,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741881_1057 (size=12151) 2024-11-19T00:13:30,269 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/db56c89a2988472591b61c1d4b7a5a49 2024-11-19T00:13:30,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/20648145c6f645fdb73d2819597562a0 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/20648145c6f645fdb73d2819597562a0 2024-11-19T00:13:30,293 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/20648145c6f645fdb73d2819597562a0, entries=150, sequenceid=187, filesize=11.9 K 2024-11-19T00:13:30,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/db31719e72fb4d18ac7ec0295612e6a6 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/db31719e72fb4d18ac7ec0295612e6a6 2024-11-19T00:13:30,309 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/db31719e72fb4d18ac7ec0295612e6a6, entries=150, sequenceid=187, filesize=11.9 K 2024-11-19T00:13:30,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/db56c89a2988472591b61c1d4b7a5a49 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/db56c89a2988472591b61c1d4b7a5a49 2024-11-19T00:13:30,327 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/db56c89a2988472591b61c1d4b7a5a49, entries=150, sequenceid=187, filesize=11.9 K 2024-11-19T00:13:30,329 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for df30878ae408737da14ed71330503a0a in 641ms, sequenceid=187, compaction requested=false 2024-11-19T00:13:30,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:30,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:30,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-19T00:13:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-19T00:13:30,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-19T00:13:30,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7290 sec 2024-11-19T00:13:30,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.7390 sec 2024-11-19T00:13:30,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on df30878ae408737da14ed71330503a0a 2024-11-19T00:13:30,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=134.18 KB heapSize=399.89 KB 2024-11-19T00:13:30,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:30,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:30,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:30,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:30,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:30,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:30,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/806478de29444313a47445e568f8ed30 is 50, key is test_row_0/A:col10/1731975210445/Put/seqid=0 2024-11-19T00:13:30,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741882_1058 (size=12151) 2024-11-19T00:13:30,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:30,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975270521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:30,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975270528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:30,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:30,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:30,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975270530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:30,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:30,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975270530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:30,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:30,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:30,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975270535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:30,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:30,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:30,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:30,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975270644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:30,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:30,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:30,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975270645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:30,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975270654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:30,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975270654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:30,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975270655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:30,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:30,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T00:13:30,754 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:13:30,754 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:13:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-19T00:13:30,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T00:13:30,760 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:13:30,761 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:13:30,762 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:13:30,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:30,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:30,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975270854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:30,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975270854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:30,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T00:13:30,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975270864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:30,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:30,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975270864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:30,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:30,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:30,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:30,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975270864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:30,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:30,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:30,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T00:13:30,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:30,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:30,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:30,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:30,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:30,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:30,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/806478de29444313a47445e568f8ed30 2024-11-19T00:13:30,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/ba7972f80d474018a278c57689a88844 is 50, key is test_row_0/B:col10/1731975210445/Put/seqid=0 2024-11-19T00:13:30,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741883_1059 (size=12151) 2024-11-19T00:13:30,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/ba7972f80d474018a278c57689a88844 2024-11-19T00:13:30,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/15b87fb4d8514f93b0e85aa93c83c389 is 50, key is test_row_0/C:col10/1731975210445/Put/seqid=0 2024-11-19T00:13:30,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741884_1060 (size=12151) 2024-11-19T00:13:30,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/15b87fb4d8514f93b0e85aa93c83c389 2024-11-19T00:13:31,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/806478de29444313a47445e568f8ed30 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/806478de29444313a47445e568f8ed30 2024-11-19T00:13:31,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/806478de29444313a47445e568f8ed30, entries=150, sequenceid=212, filesize=11.9 K 2024-11-19T00:13:31,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/ba7972f80d474018a278c57689a88844 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/ba7972f80d474018a278c57689a88844 2024-11-19T00:13:31,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/ba7972f80d474018a278c57689a88844, entries=150, sequenceid=212, filesize=11.9 K 2024-11-19T00:13:31,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/15b87fb4d8514f93b0e85aa93c83c389 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/15b87fb4d8514f93b0e85aa93c83c389 2024-11-19T00:13:31,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/15b87fb4d8514f93b0e85aa93c83c389, entries=150, sequenceid=212, filesize=11.9 K 2024-11-19T00:13:31,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~399.84 KB/409440, currentSize=46.96 KB/48090 for df30878ae408737da14ed71330503a0a in 554ms, sequenceid=212, compaction requested=true 2024-11-19T00:13:31,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:31,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:13:31,053 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:31,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:31,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:13:31,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:31,054 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:31,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:13:31,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:31,055 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:31,055 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:31,056 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/A is initiating minor compaction (all files) 2024-11-19T00:13:31,056 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/B is initiating minor compaction (all files) 2024-11-19T00:13:31,056 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/A in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:31,056 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/B in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:31,056 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/5afbe921fddc40cab85e14aa88283251, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/db31719e72fb4d18ac7ec0295612e6a6, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/ba7972f80d474018a278c57689a88844] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=36.0 K 2024-11-19T00:13:31,056 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/30b65ad1b841437a81b1547ec91c36c4, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/20648145c6f645fdb73d2819597562a0, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/806478de29444313a47445e568f8ed30] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=36.0 K 2024-11-19T00:13:31,056 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 5afbe921fddc40cab85e14aa88283251, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731975207420 2024-11-19T00:13:31,056 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30b65ad1b841437a81b1547ec91c36c4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731975207420 2024-11-19T00:13:31,057 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting db31719e72fb4d18ac7ec0295612e6a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731975208606 2024-11-19T00:13:31,057 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20648145c6f645fdb73d2819597562a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731975208606 2024-11-19T00:13:31,057 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting ba7972f80d474018a278c57689a88844, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731975209794 2024-11-19T00:13:31,058 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 806478de29444313a47445e568f8ed30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731975209794 2024-11-19T00:13:31,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T00:13:31,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:31,072 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T00:13:31,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:31,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:31,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:31,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:31,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:31,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T00:13:31,077 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#B#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:31,078 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/698d110b45d04a61b7430dc875df8647 is 50, key is test_row_0/B:col10/1731975210445/Put/seqid=0 2024-11-19T00:13:31,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/5a714035d6bd4c02b0ac2de72005bb4b is 50, key is test_row_0/A:col10/1731975210527/Put/seqid=0 2024-11-19T00:13:31,097 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#A#compaction#50 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:31,098 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/d7066e5d425946edb882f10a47b51ff7 is 50, key is test_row_0/A:col10/1731975210445/Put/seqid=0 2024-11-19T00:13:31,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741885_1061 (size=12697) 2024-11-19T00:13:31,122 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/698d110b45d04a61b7430dc875df8647 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/698d110b45d04a61b7430dc875df8647 2024-11-19T00:13:31,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741887_1063 (size=12697) 2024-11-19T00:13:31,135 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df30878ae408737da14ed71330503a0a/B of df30878ae408737da14ed71330503a0a into 698d110b45d04a61b7430dc875df8647(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:31,135 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:31,136 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/B, priority=13, startTime=1731975211053; duration=0sec 2024-11-19T00:13:31,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741886_1062 (size=12151) 2024-11-19T00:13:31,136 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:31,136 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:B 2024-11-19T00:13:31,136 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:31,140 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:31,140 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/C is initiating minor compaction (all files) 2024-11-19T00:13:31,140 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/C in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:31,140 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/a83b5b44f6e5432791395af0de664f38, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/db56c89a2988472591b61c1d4b7a5a49, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/15b87fb4d8514f93b0e85aa93c83c389] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=36.0 K 2024-11-19T00:13:31,144 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting a83b5b44f6e5432791395af0de664f38, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731975207420 2024-11-19T00:13:31,145 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting db56c89a2988472591b61c1d4b7a5a49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731975208606 2024-11-19T00:13:31,145 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 15b87fb4d8514f93b0e85aa93c83c389, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731975209794 2024-11-19T00:13:31,147 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/d7066e5d425946edb882f10a47b51ff7 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/d7066e5d425946edb882f10a47b51ff7 2024-11-19T00:13:31,160 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df30878ae408737da14ed71330503a0a/A of df30878ae408737da14ed71330503a0a into d7066e5d425946edb882f10a47b51ff7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:31,160 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:31,160 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/A, priority=13, startTime=1731975211053; duration=0sec 2024-11-19T00:13:31,161 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:31,161 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:A 2024-11-19T00:13:31,164 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#C#compaction#51 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:31,164 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/5d63f79b38fa42fbb6efb8da1cdeb1b2 is 50, key is test_row_0/C:col10/1731975210445/Put/seqid=0 2024-11-19T00:13:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on df30878ae408737da14ed71330503a0a 2024-11-19T00:13:31,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:31,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741888_1064 (size=12697) 2024-11-19T00:13:31,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731975211185, max timestamp=1731975211198Num uniques 859; 2024-11-19T00:13:31,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731975211185, max timestamp=1731975211198Num uniques 859; 2024-11-19T00:13:31,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T00:13:31,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T00:13:31,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731975211185, max timestamp=1731975211198Num uniques 859; 2024-11-19T00:13:31,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T00:13:31,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:31,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975271233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:31,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:31,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975271235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:31,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:31,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:31,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:31,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975271238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:31,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:31,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:31,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975271238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:31,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:31,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975271241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:31,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:31,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:31,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975271344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:31,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:31,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:31,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975271354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:31,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:31,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975271355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:31,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:31,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975271355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:31,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975271355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T00:13:31,537 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/5a714035d6bd4c02b0ac2de72005bb4b 2024-11-19T00:13:31,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/919a43ae3b89464c8c97ab87fa2519b6 is 50, key is test_row_0/B:col10/1731975210527/Put/seqid=0 2024-11-19T00:13:31,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741889_1065 (size=12151) 2024-11-19T00:13:31,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975271554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:31,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:31,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975271563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975271563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:31,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975271564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975271564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:31,584 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/5d63f79b38fa42fbb6efb8da1cdeb1b2 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/5d63f79b38fa42fbb6efb8da1cdeb1b2 2024-11-19T00:13:31,592 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df30878ae408737da14ed71330503a0a/C of df30878ae408737da14ed71330503a0a into 5d63f79b38fa42fbb6efb8da1cdeb1b2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:31,592 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:31,592 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/C, priority=13, startTime=1731975211054; duration=0sec 2024-11-19T00:13:31,593 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:31,593 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:C 2024-11-19T00:13:31,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:31,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975271864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:31,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:31,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:31,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:31,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975271874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:31,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975271874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:31,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:31,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:31,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:31,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:31,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975271874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:31,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975271874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:31,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:31,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:31,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:31,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T00:13:31,956 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/919a43ae3b89464c8c97ab87fa2519b6 2024-11-19T00:13:31,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/db42e3066f704e0f9dbc4974ca450dce is 50, key is test_row_0/C:col10/1731975210527/Put/seqid=0 2024-11-19T00:13:31,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741890_1066 (size=12151) 2024-11-19T00:13:31,988 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/db42e3066f704e0f9dbc4974ca450dce 2024-11-19T00:13:31,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/5a714035d6bd4c02b0ac2de72005bb4b as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/5a714035d6bd4c02b0ac2de72005bb4b 2024-11-19T00:13:32,005 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/5a714035d6bd4c02b0ac2de72005bb4b, entries=150, sequenceid=222, filesize=11.9 K 2024-11-19T00:13:32,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/919a43ae3b89464c8c97ab87fa2519b6 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/919a43ae3b89464c8c97ab87fa2519b6 2024-11-19T00:13:32,014 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/919a43ae3b89464c8c97ab87fa2519b6, entries=150, sequenceid=222, filesize=11.9 K 2024-11-19T00:13:32,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/db42e3066f704e0f9dbc4974ca450dce as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/db42e3066f704e0f9dbc4974ca450dce 2024-11-19T00:13:32,022 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/db42e3066f704e0f9dbc4974ca450dce, entries=150, sequenceid=222, filesize=11.9 K 2024-11-19T00:13:32,023 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for df30878ae408737da14ed71330503a0a in 952ms, sequenceid=222, compaction requested=false 2024-11-19T00:13:32,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:32,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:32,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-19T00:13:32,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-19T00:13:32,027 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-19T00:13:32,028 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2630 sec 2024-11-19T00:13:32,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.2730 sec 2024-11-19T00:13:32,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on df30878ae408737da14ed71330503a0a 2024-11-19T00:13:32,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-19T00:13:32,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:32,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:32,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:32,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:32,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:32,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:32,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/62fda2e1f7ac4864a23d69e33448db33 is 50, key is test_row_0/A:col10/1731975211236/Put/seqid=0 2024-11-19T00:13:32,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741891_1067 (size=12301) 2024-11-19T00:13:32,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/62fda2e1f7ac4864a23d69e33448db33 2024-11-19T00:13:32,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975272402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:32,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:32,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:32,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975272404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:32,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:32,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:32,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975272404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975272407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:32,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:32,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975272409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:32,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:32,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/c6a24eda17f2411094d31835e6e9e576 is 50, key is test_row_0/B:col10/1731975211236/Put/seqid=0 2024-11-19T00:13:32,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:32,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:32,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741892_1068 (size=12301) 2024-11-19T00:13:32,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:32,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975272514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:32,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:32,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:32,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975272524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:32,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:32,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:32,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975272524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:32,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:32,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:32,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975272524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:32,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:32,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:32,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975272524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:32,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:32,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975272723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:32,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:32,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:32,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975272734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:32,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:32,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975272734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:32,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975272734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:32,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:32,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975272734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:32,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:32,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:32,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:32,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/c6a24eda17f2411094d31835e6e9e576 2024-11-19T00:13:32,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/6be1c1d52fe84884ba17b6461798ef08 is 50, key is test_row_0/C:col10/1731975211236/Put/seqid=0 2024-11-19T00:13:32,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741893_1069 (size=12301) 2024-11-19T00:13:32,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T00:13:32,904 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:13:32,905 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:13:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-19T00:13:32,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T00:13:32,908 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:13:32,910 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:13:32,911 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:13:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T00:13:33,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975273033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:33,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:33,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:33,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975273044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:33,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:33,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:33,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975273045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:33,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:33,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975273045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:33,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:33,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:33,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975273045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:33,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:33,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T00:13:33,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:33,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:33,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:33,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:33,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:33,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T00:13:33,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:33,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:33,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:33,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:33,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:33,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:33,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T00:13:33,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/6be1c1d52fe84884ba17b6461798ef08 2024-11-19T00:13:33,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/62fda2e1f7ac4864a23d69e33448db33 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/62fda2e1f7ac4864a23d69e33448db33 2024-11-19T00:13:33,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/62fda2e1f7ac4864a23d69e33448db33, entries=150, sequenceid=261, filesize=12.0 K 2024-11-19T00:13:33,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/c6a24eda17f2411094d31835e6e9e576 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/c6a24eda17f2411094d31835e6e9e576 2024-11-19T00:13:33,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/c6a24eda17f2411094d31835e6e9e576, entries=150, sequenceid=261, filesize=12.0 K 2024-11-19T00:13:33,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/6be1c1d52fe84884ba17b6461798ef08 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/6be1c1d52fe84884ba17b6461798ef08 2024-11-19T00:13:33,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/6be1c1d52fe84884ba17b6461798ef08, entries=150, sequenceid=261, filesize=12.0 K 2024-11-19T00:13:33,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for df30878ae408737da14ed71330503a0a in 929ms, sequenceid=261, compaction requested=true 2024-11-19T00:13:33,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:33,308 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:33,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:13:33,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:33,309 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:33,310 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:33,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:13:33,310 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/A is initiating minor compaction (all files) 2024-11-19T00:13:33,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:33,310 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/A in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:33,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:13:33,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:33,310 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/d7066e5d425946edb882f10a47b51ff7, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/5a714035d6bd4c02b0ac2de72005bb4b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/62fda2e1f7ac4864a23d69e33448db33] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=36.3 K 2024-11-19T00:13:33,310 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:33,311 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/B is initiating minor compaction (all files) 2024-11-19T00:13:33,311 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/B in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:33,311 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7066e5d425946edb882f10a47b51ff7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731975209794 2024-11-19T00:13:33,311 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/698d110b45d04a61b7430dc875df8647, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/919a43ae3b89464c8c97ab87fa2519b6, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/c6a24eda17f2411094d31835e6e9e576] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=36.3 K 2024-11-19T00:13:33,311 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a714035d6bd4c02b0ac2de72005bb4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731975210511 2024-11-19T00:13:33,311 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 698d110b45d04a61b7430dc875df8647, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731975209794 2024-11-19T00:13:33,315 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62fda2e1f7ac4864a23d69e33448db33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731975211236 2024-11-19T00:13:33,317 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 919a43ae3b89464c8c97ab87fa2519b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731975210511 2024-11-19T00:13:33,319 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting c6a24eda17f2411094d31835e6e9e576, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731975211236 2024-11-19T00:13:33,330 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#B#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:33,330 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#A#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:33,331 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/3b974520699a4bde9820b9bd0a76183c is 50, key is test_row_0/A:col10/1731975211236/Put/seqid=0 2024-11-19T00:13:33,334 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/711dae44ef9d4240a2bbe3121d2f79a8 is 50, key is test_row_0/B:col10/1731975211236/Put/seqid=0 2024-11-19T00:13:33,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741894_1070 (size=12949) 2024-11-19T00:13:33,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741895_1071 (size=12949) 2024-11-19T00:13:33,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T00:13:33,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:33,376 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-19T00:13:33,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:33,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:33,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:33,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:33,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:33,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:33,377 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/711dae44ef9d4240a2bbe3121d2f79a8 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/711dae44ef9d4240a2bbe3121d2f79a8 2024-11-19T00:13:33,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/681f08ca588b455bb791c074621ac5f4 is 50, key is test_row_0/A:col10/1731975212385/Put/seqid=0 2024-11-19T00:13:33,413 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df30878ae408737da14ed71330503a0a/B of df30878ae408737da14ed71330503a0a into 711dae44ef9d4240a2bbe3121d2f79a8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:33,413 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:33,413 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/B, priority=13, startTime=1731975213308; duration=0sec 2024-11-19T00:13:33,413 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:33,413 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:B 2024-11-19T00:13:33,413 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:33,415 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:33,415 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/C is initiating minor compaction (all files) 2024-11-19T00:13:33,415 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/C in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:33,415 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/5d63f79b38fa42fbb6efb8da1cdeb1b2, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/db42e3066f704e0f9dbc4974ca450dce, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/6be1c1d52fe84884ba17b6461798ef08] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=36.3 K 2024-11-19T00:13:33,416 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d63f79b38fa42fbb6efb8da1cdeb1b2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731975209794 2024-11-19T00:13:33,418 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting db42e3066f704e0f9dbc4974ca450dce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731975210511 2024-11-19T00:13:33,419 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 6be1c1d52fe84884ba17b6461798ef08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731975211236 2024-11-19T00:13:33,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741896_1072 (size=12301) 2024-11-19T00:13:33,443 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/681f08ca588b455bb791c074621ac5f4 2024-11-19T00:13:33,456 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#C#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:33,457 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/b6ff8e9e485d4acdb2b45a6b65452fe6 is 50, key is test_row_0/C:col10/1731975211236/Put/seqid=0 2024-11-19T00:13:33,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/de2c5bd856474bb1a0e96ff658779654 is 50, key is test_row_0/B:col10/1731975212385/Put/seqid=0 2024-11-19T00:13:33,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741898_1074 (size=12949) 2024-11-19T00:13:33,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741897_1073 (size=12301) 2024-11-19T00:13:33,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T00:13:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on df30878ae408737da14ed71330503a0a 2024-11-19T00:13:33,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:33,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731975213568, max timestamp=1731975213582Num uniques 859; 2024-11-19T00:13:33,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731975213568, max timestamp=1731975213582Num uniques 859; 2024-11-19T00:13:33,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731975213568, max timestamp=1731975213582Num uniques 859; 2024-11-19T00:13:33,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T00:13:33,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T00:13:33,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T00:13:33,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:33,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:33,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975273668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:33,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975273665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:33,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:33,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975273668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:33,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:33,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:33,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:33,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:33,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975273680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:33,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:33,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:33,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975273681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:33,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:33,760 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/3b974520699a4bde9820b9bd0a76183c as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/3b974520699a4bde9820b9bd0a76183c 2024-11-19T00:13:33,772 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df30878ae408737da14ed71330503a0a/A of df30878ae408737da14ed71330503a0a into 3b974520699a4bde9820b9bd0a76183c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:33,772 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:33,772 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/A, priority=13, startTime=1731975213308; duration=0sec 2024-11-19T00:13:33,772 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:33,772 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:A 2024-11-19T00:13:33,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:33,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975273783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:33,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:33,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975273784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:33,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:33,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:33,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975273794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:33,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:33,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:33,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975273794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:33,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:33,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:33,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975273794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:33,869 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/de2c5bd856474bb1a0e96ff658779654 2024-11-19T00:13:33,872 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/b6ff8e9e485d4acdb2b45a6b65452fe6 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/b6ff8e9e485d4acdb2b45a6b65452fe6 2024-11-19T00:13:33,885 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df30878ae408737da14ed71330503a0a/C of df30878ae408737da14ed71330503a0a into b6ff8e9e485d4acdb2b45a6b65452fe6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:33,885 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:33,885 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/C, priority=13, startTime=1731975213310; duration=0sec 2024-11-19T00:13:33,885 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:33,885 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:C 2024-11-19T00:13:33,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/1b2a1e8f1e9b49e38ddd0fdb9ea5ee47 is 50, key is test_row_0/C:col10/1731975212385/Put/seqid=0 2024-11-19T00:13:33,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741899_1075 (size=12301) 2024-11-19T00:13:33,902 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/1b2a1e8f1e9b49e38ddd0fdb9ea5ee47 2024-11-19T00:13:33,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/681f08ca588b455bb791c074621ac5f4 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/681f08ca588b455bb791c074621ac5f4 2024-11-19T00:13:33,919 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/681f08ca588b455bb791c074621ac5f4, entries=150, sequenceid=270, filesize=12.0 K 2024-11-19T00:13:33,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/de2c5bd856474bb1a0e96ff658779654 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/de2c5bd856474bb1a0e96ff658779654 2024-11-19T00:13:33,929 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/de2c5bd856474bb1a0e96ff658779654, entries=150, sequenceid=270, filesize=12.0 K 2024-11-19T00:13:33,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/1b2a1e8f1e9b49e38ddd0fdb9ea5ee47 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/1b2a1e8f1e9b49e38ddd0fdb9ea5ee47 2024-11-19T00:13:33,938 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/1b2a1e8f1e9b49e38ddd0fdb9ea5ee47, entries=150, sequenceid=270, filesize=12.0 K 2024-11-19T00:13:33,940 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=147.60 KB/151140 for df30878ae408737da14ed71330503a0a in 564ms, sequenceid=270, compaction requested=false 2024-11-19T00:13:33,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:33,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:33,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-19T00:13:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-19T00:13:33,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-19T00:13:33,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0330 sec 2024-11-19T00:13:33,950 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.0420 sec 2024-11-19T00:13:34,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on df30878ae408737da14ed71330503a0a 2024-11-19T00:13:34,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=154.31 KB heapSize=426 KB 2024-11-19T00:13:34,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:34,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:34,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:34,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:34,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:34,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:34,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/f4c4db175c7a481eb280758a368a042d is 50, key is test_row_0/A:col10/1731975213677/Put/seqid=0 2024-11-19T00:13:34,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975274004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:34,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:34,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975274004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:34,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:34,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975274006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:34,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:34,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:34,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:34,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975274009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:34,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:34,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:34,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975274011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:34,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:34,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741900_1076 (size=14741) 2024-11-19T00:13:34,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T00:13:34,044 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:13:34,045 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:13:34,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-19T00:13:34,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T00:13:34,048 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:13:34,049 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:13:34,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:13:34,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:34,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975274123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:34,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:34,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:34,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975274124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:34,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:34,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:34,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975274124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:34,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:34,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975274124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:34,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T00:13:34,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T00:13:34,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:34,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:34,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:34,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:34,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:34,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:34,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:34,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975274324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:34,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:34,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975274335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:34,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:34,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:34,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975274335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:34,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:34,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:34,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975274335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:34,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:34,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:34,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975274344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:34,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T00:13:34,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:34,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:34,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:34,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:34,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:34,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:34,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T00:13:34,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/f4c4db175c7a481eb280758a368a042d 2024-11-19T00:13:34,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/f300d8eb38cd4c8f9ecd7deb3048b413 is 50, key is test_row_0/B:col10/1731975213677/Put/seqid=0 2024-11-19T00:13:34,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741901_1077 (size=12301) 2024-11-19T00:13:34,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/f300d8eb38cd4c8f9ecd7deb3048b413 2024-11-19T00:13:34,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/54788d241c3f4540ae45941418745b14 is 50, key is test_row_0/C:col10/1731975213677/Put/seqid=0 2024-11-19T00:13:34,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741902_1078 (size=12301) 2024-11-19T00:13:34,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T00:13:34,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:34,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:34,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:34,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:34,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:34,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:34,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:34,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975274644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:34,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:34,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975274644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:34,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:34,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:34,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:34,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975274645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:34,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:34,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975274654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:34,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:34,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T00:13:34,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:34,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:34,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:34,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:34,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T00:13:34,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T00:13:34,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:34,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:34,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:34,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:34,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:34,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975274834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:34,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:34,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:34,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/54788d241c3f4540ae45941418745b14 2024-11-19T00:13:34,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/f4c4db175c7a481eb280758a368a042d as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/f4c4db175c7a481eb280758a368a042d 2024-11-19T00:13:34,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/f4c4db175c7a481eb280758a368a042d, entries=200, sequenceid=310, filesize=14.4 K 2024-11-19T00:13:34,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/f300d8eb38cd4c8f9ecd7deb3048b413 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/f300d8eb38cd4c8f9ecd7deb3048b413 2024-11-19T00:13:34,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/f300d8eb38cd4c8f9ecd7deb3048b413, entries=150, sequenceid=310, filesize=12.0 K 2024-11-19T00:13:34,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/54788d241c3f4540ae45941418745b14 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/54788d241c3f4540ae45941418745b14 2024-11-19T00:13:34,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/54788d241c3f4540ae45941418745b14, entries=150, sequenceid=310, filesize=12.0 K 2024-11-19T00:13:34,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~445.88 KB/456576, currentSize=26.84 KB/27480 for df30878ae408737da14ed71330503a0a in 912ms, sequenceid=310, compaction requested=true 2024-11-19T00:13:34,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:34,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:13:34,913 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:34,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:34,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:13:34,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:34,913 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:34,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:13:34,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:34,914 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:34,914 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:34,914 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/B is initiating minor compaction (all files) 2024-11-19T00:13:34,914 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/A is initiating minor compaction (all files) 2024-11-19T00:13:34,914 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/A in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:34,914 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/B in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:34,915 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/711dae44ef9d4240a2bbe3121d2f79a8, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/de2c5bd856474bb1a0e96ff658779654, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/f300d8eb38cd4c8f9ecd7deb3048b413] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=36.7 K 2024-11-19T00:13:34,915 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/3b974520699a4bde9820b9bd0a76183c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/681f08ca588b455bb791c074621ac5f4, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/f4c4db175c7a481eb280758a368a042d] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=39.1 K 2024-11-19T00:13:34,915 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b974520699a4bde9820b9bd0a76183c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731975211236 2024-11-19T00:13:34,915 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 711dae44ef9d4240a2bbe3121d2f79a8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731975211236 2024-11-19T00:13:34,915 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting de2c5bd856474bb1a0e96ff658779654, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731975212385 2024-11-19T00:13:34,915 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 681f08ca588b455bb791c074621ac5f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731975212385 2024-11-19T00:13:34,916 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting f300d8eb38cd4c8f9ecd7deb3048b413, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731975213677 2024-11-19T00:13:34,916 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4c4db175c7a481eb280758a368a042d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731975213677 2024-11-19T00:13:34,930 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#B#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:34,930 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#A#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:34,930 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/b462be64fe3e4a07bc2fcd8c7c6933db is 50, key is test_row_0/A:col10/1731975213677/Put/seqid=0 2024-11-19T00:13:34,930 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/e01d4ed838864ac59add774c7a6b37b4 is 50, key is test_row_0/B:col10/1731975213677/Put/seqid=0 2024-11-19T00:13:34,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741904_1080 (size=13051) 2024-11-19T00:13:34,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741903_1079 (size=13051) 2024-11-19T00:13:34,952 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/b462be64fe3e4a07bc2fcd8c7c6933db as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/b462be64fe3e4a07bc2fcd8c7c6933db 2024-11-19T00:13:34,958 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/e01d4ed838864ac59add774c7a6b37b4 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/e01d4ed838864ac59add774c7a6b37b4 2024-11-19T00:13:34,966 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df30878ae408737da14ed71330503a0a/A of df30878ae408737da14ed71330503a0a into b462be64fe3e4a07bc2fcd8c7c6933db(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:34,966 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:34,966 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/A, priority=13, startTime=1731975214913; duration=0sec 2024-11-19T00:13:34,967 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:34,967 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:A 2024-11-19T00:13:34,967 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:34,969 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df30878ae408737da14ed71330503a0a/B of df30878ae408737da14ed71330503a0a into e01d4ed838864ac59add774c7a6b37b4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:34,969 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:34,969 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/B, priority=13, startTime=1731975214913; duration=0sec 2024-11-19T00:13:34,969 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:34,969 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:B 2024-11-19T00:13:34,970 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:34,970 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/C is initiating minor compaction (all files) 2024-11-19T00:13:34,970 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/C in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:34,970 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/b6ff8e9e485d4acdb2b45a6b65452fe6, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/1b2a1e8f1e9b49e38ddd0fdb9ea5ee47, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/54788d241c3f4540ae45941418745b14] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=36.7 K 2024-11-19T00:13:34,971 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6ff8e9e485d4acdb2b45a6b65452fe6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731975211236 2024-11-19T00:13:34,972 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b2a1e8f1e9b49e38ddd0fdb9ea5ee47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731975212385 2024-11-19T00:13:34,973 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54788d241c3f4540ae45941418745b14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731975213677 2024-11-19T00:13:34,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T00:13:34,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:34,976 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-19T00:13:34,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:34,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:34,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:34,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:34,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:34,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:34,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/2ecc1d78faf2423ea8372bffd92e2358 is 50, key is test_row_1/A:col10/1731975214005/Put/seqid=0 2024-11-19T00:13:35,020 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#C#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:35,020 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/ad58a0031e5c488aba139ce30c083c94 is 50, key is test_row_0/C:col10/1731975213677/Put/seqid=0 2024-11-19T00:13:35,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741905_1081 (size=9857) 2024-11-19T00:13:35,028 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/2ecc1d78faf2423ea8372bffd92e2358 2024-11-19T00:13:35,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741906_1082 (size=13051) 2024-11-19T00:13:35,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/b8107d8391864d2189271e2affa876f4 is 50, key is test_row_1/B:col10/1731975214005/Put/seqid=0 2024-11-19T00:13:35,052 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/ad58a0031e5c488aba139ce30c083c94 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/ad58a0031e5c488aba139ce30c083c94 2024-11-19T00:13:35,064 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df30878ae408737da14ed71330503a0a/C of df30878ae408737da14ed71330503a0a into ad58a0031e5c488aba139ce30c083c94(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:35,064 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:35,064 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/C, priority=13, startTime=1731975214913; duration=0sec 2024-11-19T00:13:35,064 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:35,064 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:C 2024-11-19T00:13:35,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741907_1083 (size=9857) 2024-11-19T00:13:35,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on df30878ae408737da14ed71330503a0a 2024-11-19T00:13:35,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T00:13:35,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975215169, max timestamp=1731975215188Num uniques 859; 2024-11-19T00:13:35,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975215169, max timestamp=1731975215188Num uniques 859; 2024-11-19T00:13:35,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:13:35,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:13:35,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975215169, max timestamp=1731975215188Num uniques 859; 2024-11-19T00:13:35,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:13:35,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731975215207, max timestamp=1731975215220Num uniques 1068; 2024-11-19T00:13:35,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-19T00:13:35,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731975215207, max timestamp=1731975215220Num uniques 1068; 2024-11-19T00:13:35,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731975215207, max timestamp=1731975215220Num uniques 1068; 2024-11-19T00:13:35,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-19T00:13:35,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-19T00:13:35,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:35,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975275260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975275260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:35,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1731975215230, max timestamp=1731975215264Num uniques 1177; 2024-11-19T00:13:35,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1731975215230, max timestamp=1731975215264Num uniques 1177; 2024-11-19T00:13:35,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=17262, suffix heap size=150376,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=377 2024-11-19T00:13:35,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=17262, suffix heap size=150376,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=377 2024-11-19T00:13:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:35,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731975215257, max timestamp=1731975215264Num uniques 1177; 2024-11-19T00:13:35,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:35,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=8102, suffix heap size=150376,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=177 2024-11-19T00:13:35,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731975215303, max timestamp=1731975215330Num uniques 1236; 2024-11-19T00:13:35,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731975215303, max timestamp=1731975215330Num uniques 1236; 2024-11-19T00:13:35,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=56606, new segment data size=10806, suffix heap size=156512,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1236, new segment cells count=236 2024-11-19T00:13:35,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=56606, new segment data size=10806, suffix heap size=156512,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1236, new segment cells count=236 2024-11-19T00:13:35,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731975215316, max timestamp=1731975215330Num uniques 1036; 2024-11-19T00:13:35,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-19T00:13:35,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975275384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:35,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13500, totalHeapSize=30920, min timestamp=1731975215362, max timestamp=1731975215387Num uniques 1095; 2024-11-19T00:13:35,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13500, totalHeapSize=30920, min timestamp=1731975215362, max timestamp=1731975215387Num uniques 1095; 2024-11-19T00:13:35,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13500, totalHeapSize=30920, min timestamp=1731975215362, max timestamp=1731975215387Num uniques 1045; 2024-11-19T00:13:35,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50140, new segment data size=13500, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-19T00:13:35,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50140, new segment data size=13500, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-19T00:13:35,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47850, new segment data size=13500, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-19T00:13:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:35,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975275436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:35,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:35,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975275438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:35,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:35,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975275439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:35,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:35,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:35,478 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/b8107d8391864d2189271e2affa876f4 2024-11-19T00:13:35,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/76ebcf67772e4316b89d791f6c63bd4d is 50, key is test_row_1/C:col10/1731975214005/Put/seqid=0 2024-11-19T00:13:35,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741908_1084 (size=9857) 2024-11-19T00:13:35,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975275503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:35,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975275544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:35,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:35,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975275554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:35,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:35,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975275554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:35,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:35,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:35,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975275714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:35,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:35,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975275753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:35,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:35,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975275764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:35,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:35,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975275764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:35,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:35,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975275843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:35,847 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_0' in df30878ae408737da14ed71330503a0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:13:35,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:35,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:35,902 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/76ebcf67772e4316b89d791f6c63bd4d 2024-11-19T00:13:35,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/2ecc1d78faf2423ea8372bffd92e2358 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/2ecc1d78faf2423ea8372bffd92e2358 2024-11-19T00:13:35,919 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/2ecc1d78faf2423ea8372bffd92e2358, entries=100, sequenceid=319, filesize=9.6 K 2024-11-19T00:13:35,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/b8107d8391864d2189271e2affa876f4 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/b8107d8391864d2189271e2affa876f4 2024-11-19T00:13:35,925 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/b8107d8391864d2189271e2affa876f4, entries=100, sequenceid=319, filesize=9.6 K 2024-11-19T00:13:35,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/76ebcf67772e4316b89d791f6c63bd4d as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/76ebcf67772e4316b89d791f6c63bd4d 2024-11-19T00:13:35,937 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/76ebcf67772e4316b89d791f6c63bd4d, entries=100, sequenceid=319, filesize=9.6 K 2024-11-19T00:13:35,938 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=154.31 KB/158010 for df30878ae408737da14ed71330503a0a in 962ms, sequenceid=319, compaction requested=false 2024-11-19T00:13:35,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:35,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:35,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-19T00:13:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-19T00:13:35,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-19T00:13:35,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8910 sec 2024-11-19T00:13:35,948 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.9010 sec 2024-11-19T00:13:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on df30878ae408737da14ed71330503a0a 2024-11-19T00:13:36,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731975215425, max timestamp=1731975216026Num uniques 1154; 2024-11-19T00:13:36,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=161.02 KB heapSize=452.63 KB 2024-11-19T00:13:36,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-11-19T00:13:36,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:36,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:36,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:36,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731975215425, max timestamp=1731975216026Num uniques 1154; 2024-11-19T00:13:36,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-11-19T00:13:36,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731975215425, max timestamp=1731975216026Num uniques 1154; 2024-11-19T00:13:36,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:36,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:36,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-11-19T00:13:36,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:36,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/6fcbcf84028744019dfaf1339b34f419 is 50, key is test_row_0/A:col10/1731975215437/Put/seqid=0 2024-11-19T00:13:36,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741909_1085 (size=14741) 2024-11-19T00:13:36,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/6fcbcf84028744019dfaf1339b34f419 2024-11-19T00:13:36,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/8df1d511f5ab4359a22b7101867b0684 is 50, key is test_row_0/B:col10/1731975215437/Put/seqid=0 2024-11-19T00:13:36,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741910_1086 (size=12301) 2024-11-19T00:13:36,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731975216068, max timestamp=1731975216118Num uniques 859; 2024-11-19T00:13:36,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731975216068, max timestamp=1731975216118Num uniques 859; 2024-11-19T00:13:36,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T00:13:36,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T00:13:36,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975216086, max timestamp=1731975216118Num uniques 859; 2024-11-19T00:13:36,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:13:36,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731975216139, max timestamp=1731975216156Num uniques 1168; 2024-11-19T00:13:36,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731975216139, max timestamp=1731975216156Num uniques 1168; 2024-11-19T00:13:36,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731975216139, max timestamp=1731975216156Num uniques 1018; 2024-11-19T00:13:36,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-11-19T00:13:36,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T00:13:36,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-11-19T00:13:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T00:13:36,195 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:13:36,196 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:13:36,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-19T00:13:36,200 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:13:36,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731975216185, max timestamp=1731975216196Num uniques 1077; 2024-11-19T00:13:36,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-19T00:13:36,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731975216185, max timestamp=1731975216196Num uniques 1077; 2024-11-19T00:13:36,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-19T00:13:36,204 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:13:36,204 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:13:36,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T00:13:36,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731975216185, max timestamp=1731975216196Num uniques 1077; 2024-11-19T00:13:36,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-19T00:13:36,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=486, cellSize=22256, totalHeapSize=50784, min timestamp=1731975216218, max timestamp=1731975216239Num uniques 1036; 2024-11-19T00:13:36,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=486, cellSize=22256, totalHeapSize=50784, min timestamp=1731975216218, max timestamp=1731975216239Num uniques 1036; 2024-11-19T00:13:36,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=22256, suffix heap size=135712,new segment heap size=50784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=486 2024-11-19T00:13:36,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=22256, suffix heap size=135712,new segment heap size=50784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=486 2024-11-19T00:13:36,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=486, cellSize=22256, totalHeapSize=50784, min timestamp=1731975216218, max timestamp=1731975216239Num uniques 1036; 2024-11-19T00:13:36,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=22256, suffix heap size=135712,new segment heap size=50784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=486 2024-11-19T00:13:36,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975276267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:36,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:36,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975276268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:36,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:36,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:36,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:36,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975276271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:36,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:36,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:36,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975276275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:36,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T00:13:36,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T00:13:36,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:36,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:36,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:36,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:36,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:36,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:36,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975276383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:36,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975276384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:36,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975276384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:36,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:36,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975276393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:36,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:36,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/8df1d511f5ab4359a22b7101867b0684 2024-11-19T00:13:36,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/3b091b0834c34fb9bd9f99b1c4133ee6 is 50, key is test_row_0/C:col10/1731975215437/Put/seqid=0 2024-11-19T00:13:36,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741911_1087 (size=12301) 2024-11-19T00:13:36,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/3b091b0834c34fb9bd9f99b1c4133ee6 2024-11-19T00:13:36,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/6fcbcf84028744019dfaf1339b34f419 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/6fcbcf84028744019dfaf1339b34f419 2024-11-19T00:13:36,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/6fcbcf84028744019dfaf1339b34f419, entries=200, sequenceid=427, filesize=14.4 K 2024-11-19T00:13:36,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/8df1d511f5ab4359a22b7101867b0684 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/8df1d511f5ab4359a22b7101867b0684 2024-11-19T00:13:36,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T00:13:36,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:36,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:36,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:36,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:36,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:36,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/8df1d511f5ab4359a22b7101867b0684, entries=150, sequenceid=427, filesize=12.0 K 2024-11-19T00:13:36,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/3b091b0834c34fb9bd9f99b1c4133ee6 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/3b091b0834c34fb9bd9f99b1c4133ee6 2024-11-19T00:13:36,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/3b091b0834c34fb9bd9f99b1c4133ee6, entries=150, sequenceid=427, filesize=12.0 K 2024-11-19T00:13:36,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.66 KB/68256, currentSize=167.72 KB/171750 for df30878ae408737da14ed71330503a0a in 499ms, sequenceid=427, compaction requested=true 2024-11-19T00:13:36,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:36,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:13:36,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:36,530 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:36,530 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:36,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:13:36,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:36,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:13:36,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:36,531 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:36,531 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37649 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:36,531 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/A is initiating minor compaction (all files) 2024-11-19T00:13:36,531 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/A in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:36,531 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/B is initiating minor compaction (all files) 2024-11-19T00:13:36,531 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/b462be64fe3e4a07bc2fcd8c7c6933db, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/2ecc1d78faf2423ea8372bffd92e2358, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/6fcbcf84028744019dfaf1339b34f419] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=36.8 K 2024-11-19T00:13:36,531 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/B in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:36,532 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/e01d4ed838864ac59add774c7a6b37b4, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/b8107d8391864d2189271e2affa876f4, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/8df1d511f5ab4359a22b7101867b0684] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=34.4 K 2024-11-19T00:13:36,532 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting b462be64fe3e4a07bc2fcd8c7c6933db, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731975213677 2024-11-19T00:13:36,532 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting e01d4ed838864ac59add774c7a6b37b4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731975213677 2024-11-19T00:13:36,533 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ecc1d78faf2423ea8372bffd92e2358, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731975214005 2024-11-19T00:13:36,533 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting b8107d8391864d2189271e2affa876f4, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731975214005 2024-11-19T00:13:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T00:13:36,534 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 8df1d511f5ab4359a22b7101867b0684, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731975215425 2024-11-19T00:13:36,534 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fcbcf84028744019dfaf1339b34f419, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731975215425 2024-11-19T00:13:36,549 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#B#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:36,550 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/fbe35e3089cc48a78c9b25f533dc9300 is 50, key is test_row_0/B:col10/1731975215437/Put/seqid=0 2024-11-19T00:13:36,551 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#A#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:36,552 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/65d76253ef284f0b93a4fca32edd604a is 50, key is test_row_0/A:col10/1731975215437/Put/seqid=0 2024-11-19T00:13:36,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741912_1088 (size=13153) 2024-11-19T00:13:36,575 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/65d76253ef284f0b93a4fca32edd604a as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/65d76253ef284f0b93a4fca32edd604a 2024-11-19T00:13:36,584 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df30878ae408737da14ed71330503a0a/A of df30878ae408737da14ed71330503a0a into 65d76253ef284f0b93a4fca32edd604a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:36,584 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:36,584 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/A, priority=13, startTime=1731975216529; duration=0sec 2024-11-19T00:13:36,584 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:36,584 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:A 2024-11-19T00:13:36,584 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:36,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741913_1089 (size=13153) 2024-11-19T00:13:36,588 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:36,588 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/C is initiating minor compaction (all files) 2024-11-19T00:13:36,588 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/C in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:36,588 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/ad58a0031e5c488aba139ce30c083c94, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/76ebcf67772e4316b89d791f6c63bd4d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/3b091b0834c34fb9bd9f99b1c4133ee6] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=34.4 K 2024-11-19T00:13:36,589 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad58a0031e5c488aba139ce30c083c94, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731975213677 2024-11-19T00:13:36,589 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76ebcf67772e4316b89d791f6c63bd4d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731975214005 2024-11-19T00:13:36,589 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b091b0834c34fb9bd9f99b1c4133ee6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731975215425 2024-11-19T00:13:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on df30878ae408737da14ed71330503a0a 2024-11-19T00:13:36,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=174.43 KB heapSize=473.86 KB 2024-11-19T00:13:36,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:36,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:36,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:36,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:36,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:36,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:36,603 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#C#compaction#77 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:36,603 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/62ba1a4802094afaa4b56f5ad70cdc1d is 50, key is test_row_0/C:col10/1731975215437/Put/seqid=0 2024-11-19T00:13:36,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975276600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:36,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:36,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:36,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975276604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:36,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:36,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975276607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:36,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:36,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:36,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975276608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:36,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/2482d86d1f844fbaa6150686f154839c is 50, key is test_row_0/A:col10/1731975216273/Put/seqid=0 2024-11-19T00:13:36,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:36,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741914_1090 (size=13153) 2024-11-19T00:13:36,640 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/62ba1a4802094afaa4b56f5ad70cdc1d as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/62ba1a4802094afaa4b56f5ad70cdc1d 2024-11-19T00:13:36,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741915_1091 (size=12301) 2024-11-19T00:13:36,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/2482d86d1f844fbaa6150686f154839c 2024-11-19T00:13:36,655 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df30878ae408737da14ed71330503a0a/C of df30878ae408737da14ed71330503a0a into 62ba1a4802094afaa4b56f5ad70cdc1d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:36,655 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:36,655 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/C, priority=13, startTime=1731975216530; duration=0sec 2024-11-19T00:13:36,657 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:36,657 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:C 2024-11-19T00:13:36,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T00:13:36,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:36,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:36,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:36,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:36,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:36,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/1da4a672ae9b410bae2f06325ebde359 is 50, key is test_row_0/B:col10/1731975216273/Put/seqid=0 2024-11-19T00:13:36,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741916_1092 (size=12301) 2024-11-19T00:13:36,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975276713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:36,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975276724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:36,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975276724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:36,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:36,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:36,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T00:13:36,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:36,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:36,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:36,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:36,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T00:13:36,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:36,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975276923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:36,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975276923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:36,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:36,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:36,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975276934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:36,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:36,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975276934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:36,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:36,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:36,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:36,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T00:13:36,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:36,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:36,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:36,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:36,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:36,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:36,994 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/fbe35e3089cc48a78c9b25f533dc9300 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/fbe35e3089cc48a78c9b25f533dc9300 2024-11-19T00:13:37,000 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df30878ae408737da14ed71330503a0a/B of df30878ae408737da14ed71330503a0a into fbe35e3089cc48a78c9b25f533dc9300(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:37,000 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:37,001 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/B, priority=13, startTime=1731975216530; duration=0sec 2024-11-19T00:13:37,001 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:37,001 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:B 2024-11-19T00:13:37,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/1da4a672ae9b410bae2f06325ebde359 2024-11-19T00:13:37,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/86353573ced54f75822a14d13c73cd12 is 50, key is test_row_0/C:col10/1731975216273/Put/seqid=0 2024-11-19T00:13:37,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741917_1093 (size=12301) 2024-11-19T00:13:37,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T00:13:37,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:37,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:37,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:37,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:37,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:37,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:37,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975277234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:37,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:37,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:37,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:37,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975277244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:37,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:37,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:37,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:37,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975277244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:37,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:37,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:37,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:37,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T00:13:37,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:37,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:37,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:37,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:37,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T00:13:37,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:37,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975277434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:37,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:37,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T00:13:37,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:37,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:37,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:37,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:37,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:37,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:37,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:37,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:37,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/86353573ced54f75822a14d13c73cd12 2024-11-19T00:13:37,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/2482d86d1f844fbaa6150686f154839c as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/2482d86d1f844fbaa6150686f154839c 2024-11-19T00:13:37,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/2482d86d1f844fbaa6150686f154839c, entries=150, sequenceid=516, filesize=12.0 K 2024-11-19T00:13:37,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/1da4a672ae9b410bae2f06325ebde359 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/1da4a672ae9b410bae2f06325ebde359 2024-11-19T00:13:37,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/1da4a672ae9b410bae2f06325ebde359, entries=150, sequenceid=516, filesize=12.0 K 2024-11-19T00:13:37,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/86353573ced54f75822a14d13c73cd12 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/86353573ced54f75822a14d13c73cd12 2024-11-19T00:13:37,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/86353573ced54f75822a14d13c73cd12, entries=150, sequenceid=516, filesize=12.0 K 2024-11-19T00:13:37,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~473.81 KB/485184, currentSize=13.42 KB/13740 for df30878ae408737da14ed71330503a0a in 966ms, sequenceid=516, compaction requested=false 2024-11-19T00:13:37,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:37,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T00:13:37,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:37,591 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-19T00:13:37,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:37,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:37,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:37,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:37,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:37,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:37,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/d5a01195b5cb4051a6917c66dc2c934a is 50, key is test_row_0/A:col10/1731975216600/Put/seqid=0 2024-11-19T00:13:37,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741918_1094 (size=9857) 2024-11-19T00:13:37,636 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/d5a01195b5cb4051a6917c66dc2c934a 2024-11-19T00:13:37,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/4dde8e6400e9417f8006b450210507ac is 50, key is test_row_0/B:col10/1731975216600/Put/seqid=0 2024-11-19T00:13:37,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741919_1095 (size=9857) 2024-11-19T00:13:37,657 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/4dde8e6400e9417f8006b450210507ac 2024-11-19T00:13:37,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/8b309833728f482aa7cca5a431b3a723 is 50, key is test_row_0/C:col10/1731975216600/Put/seqid=0 2024-11-19T00:13:37,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741920_1096 (size=9857) 2024-11-19T00:13:37,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on df30878ae408737da14ed71330503a0a 2024-11-19T00:13:37,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:37,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975217768, max timestamp=1731975217779Num uniques 859; 2024-11-19T00:13:37,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:13:37,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975217768, max timestamp=1731975217779Num uniques 859; 2024-11-19T00:13:37,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975217768, max timestamp=1731975217779Num uniques 859; 2024-11-19T00:13:37,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:13:37,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:13:37,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731975217809, max timestamp=1731975217852Num uniques 1018; 2024-11-19T00:13:37,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731975217809, max timestamp=1731975217852Num uniques 1018; 2024-11-19T00:13:37,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T00:13:37,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T00:13:37,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731975217809, max timestamp=1731975217852Num uniques 1018; 2024-11-19T00:13:37,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T00:13:37,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1731975217872, max timestamp=1731975217897Num uniques 1077; 2024-11-19T00:13:37,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1731975217872, max timestamp=1731975217897Num uniques 1077; 2024-11-19T00:13:37,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1731975217872, max timestamp=1731975217897Num uniques 1077; 2024-11-19T00:13:37,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=17262, suffix heap size=139976,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=377 2024-11-19T00:13:37,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=17262, suffix heap size=139976,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=377 2024-11-19T00:13:37,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=17262, suffix heap size=139976,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=377 2024-11-19T00:13:37,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:37,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975277969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:37,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:37,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975277970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:37,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731975217936, max timestamp=1731975217974Num uniques 1236; 2024-11-19T00:13:37,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731975217936, max timestamp=1731975217974Num uniques 1236; 2024-11-19T00:13:37,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731975217936, max timestamp=1731975217974Num uniques 1236; 2024-11-19T00:13:37,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=56606, new segment data size=8516, suffix heap size=156512,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1236, new segment cells count=186 2024-11-19T00:13:37,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=56606, new segment data size=8516, suffix heap size=156512,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1236, new segment cells count=186 2024-11-19T00:13:37,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=56606, new segment data size=8516, suffix heap size=156512,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1236, new segment cells count=186 2024-11-19T00:13:37,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:37,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:37,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:37,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:37,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:37,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:38,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731975218037, max timestamp=1731975218060Num uniques 1045; 2024-11-19T00:13:38,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-19T00:13:38,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731975218037, max timestamp=1731975218060Num uniques 1045; 2024-11-19T00:13:38,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-19T00:13:38,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731975218037, max timestamp=1731975218060Num uniques 1045; 2024-11-19T00:13:38,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-19T00:13:38,080 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/8b309833728f482aa7cca5a431b3a723 2024-11-19T00:13:38,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/d5a01195b5cb4051a6917c66dc2c934a as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/d5a01195b5cb4051a6917c66dc2c934a 2024-11-19T00:13:38,099 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/d5a01195b5cb4051a6917c66dc2c934a, entries=100, sequenceid=523, filesize=9.6 K 2024-11-19T00:13:38,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/4dde8e6400e9417f8006b450210507ac as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/4dde8e6400e9417f8006b450210507ac 2024-11-19T00:13:38,108 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/4dde8e6400e9417f8006b450210507ac, entries=100, sequenceid=523, filesize=9.6 K 2024-11-19T00:13:38,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/8b309833728f482aa7cca5a431b3a723 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/8b309833728f482aa7cca5a431b3a723 2024-11-19T00:13:38,119 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/8b309833728f482aa7cca5a431b3a723, entries=100, sequenceid=523, filesize=9.6 K 2024-11-19T00:13:38,120 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=140.89 KB/144270 for df30878ae408737da14ed71330503a0a in 529ms, sequenceid=523, compaction requested=true 2024-11-19T00:13:38,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:38,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:38,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-19T00:13:38,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=147.60 KB heapSize=412.08 KB 2024-11-19T00:13:38,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-19T00:13:38,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:38,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:38,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:38,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:38,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:38,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:38,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on df30878ae408737da14ed71330503a0a 2024-11-19T00:13:38,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-19T00:13:38,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9180 sec 2024-11-19T00:13:38,127 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.9300 sec 2024-11-19T00:13:38,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/91991e3efadf4fff88a8d5644bcf5705 is 50, key is test_row_0/A:col10/1731975218119/Put/seqid=0 2024-11-19T00:13:38,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:38,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975278130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:38,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:38,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:38,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975278132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:38,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:38,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:38,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975278141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:38,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:38,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975278141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:38,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741921_1097 (size=17181) 2024-11-19T00:13:38,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=628 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/91991e3efadf4fff88a8d5644bcf5705 2024-11-19T00:13:38,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/84a53c525d5949a7a41c9be50e24c0dc is 50, key is test_row_0/B:col10/1731975218119/Put/seqid=0 2024-11-19T00:13:38,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741922_1098 (size=12301) 2024-11-19T00:13:38,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:38,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975278255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:38,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:38,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975278255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:38,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:38,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975278255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:38,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:38,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:38,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:38,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975278255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:38,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:38,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:38,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T00:13:38,364 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:13:38,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:13:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-19T00:13:38,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T00:13:38,366 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:13:38,367 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:13:38,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:13:38,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975278454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:38,456 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_2' in df30878ae408737da14ed71330503a0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:13:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:38,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:38,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975278463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:38,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:38,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975278463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:38,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:38,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:38,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975278464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:38,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:38,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:38,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975278464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:38,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:38,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:38,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T00:13:38,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-19T00:13:38,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:38,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:38,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:38,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:38,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:38,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:38,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=628 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/84a53c525d5949a7a41c9be50e24c0dc 2024-11-19T00:13:38,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/9acb79f993a347908341e3e0bf8321ad is 50, key is test_row_0/C:col10/1731975218119/Put/seqid=0 2024-11-19T00:13:38,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741923_1099 (size=12301) 2024-11-19T00:13:38,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-19T00:13:38,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:38,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:38,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:38,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:38,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T00:13:38,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975278773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:38,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975278774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:38,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975278774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975278774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:38,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:38,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:38,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-19T00:13:38,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:38,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:38,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:38,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:38,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:38,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:38,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-19T00:13:38,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:38,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:38,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:38,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:38,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:38,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:38,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T00:13:39,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=628 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/9acb79f993a347908341e3e0bf8321ad 2024-11-19T00:13:39,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/91991e3efadf4fff88a8d5644bcf5705 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/91991e3efadf4fff88a8d5644bcf5705 2024-11-19T00:13:39,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/91991e3efadf4fff88a8d5644bcf5705, entries=250, sequenceid=628, filesize=16.8 K 2024-11-19T00:13:39,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/84a53c525d5949a7a41c9be50e24c0dc as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/84a53c525d5949a7a41c9be50e24c0dc 2024-11-19T00:13:39,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/84a53c525d5949a7a41c9be50e24c0dc, entries=150, sequenceid=628, filesize=12.0 K 2024-11-19T00:13:39,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/9acb79f993a347908341e3e0bf8321ad as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/9acb79f993a347908341e3e0bf8321ad 2024-11-19T00:13:39,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/9acb79f993a347908341e3e0bf8321ad, entries=150, sequenceid=628, filesize=12.0 K 2024-11-19T00:13:39,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~412.03 KB/421920, currentSize=40.25 KB/41220 for df30878ae408737da14ed71330503a0a in 954ms, sequenceid=628, compaction requested=true 2024-11-19T00:13:39,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:39,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:13:39,075 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:13:39,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:39,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:13:39,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:39,075 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:13:39,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:13:39,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:39,077 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47612 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:13:39,077 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52492 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:13:39,077 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/B is initiating minor compaction (all files) 2024-11-19T00:13:39,077 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/A is initiating minor compaction (all files) 2024-11-19T00:13:39,077 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/A in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:39,077 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/B in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:39,077 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/fbe35e3089cc48a78c9b25f533dc9300, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/1da4a672ae9b410bae2f06325ebde359, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/4dde8e6400e9417f8006b450210507ac, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/84a53c525d5949a7a41c9be50e24c0dc] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=46.5 K 2024-11-19T00:13:39,077 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/65d76253ef284f0b93a4fca32edd604a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/2482d86d1f844fbaa6150686f154839c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/d5a01195b5cb4051a6917c66dc2c934a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/91991e3efadf4fff88a8d5644bcf5705] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=51.3 K 2024-11-19T00:13:39,077 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting fbe35e3089cc48a78c9b25f533dc9300, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731975215425 2024-11-19T00:13:39,077 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65d76253ef284f0b93a4fca32edd604a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731975215425 2024-11-19T00:13:39,078 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 1da4a672ae9b410bae2f06325ebde359, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1731975216264 2024-11-19T00:13:39,078 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2482d86d1f844fbaa6150686f154839c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1731975216264 2024-11-19T00:13:39,078 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dde8e6400e9417f8006b450210507ac, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731975216600 2024-11-19T00:13:39,078 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5a01195b5cb4051a6917c66dc2c934a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731975216600 2024-11-19T00:13:39,078 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 84a53c525d5949a7a41c9be50e24c0dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=628, earliestPutTs=1731975218109 2024-11-19T00:13:39,079 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91991e3efadf4fff88a8d5644bcf5705, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=628, earliestPutTs=1731975218097 2024-11-19T00:13:39,100 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#B#compaction#87 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:39,100 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/50eeed67f8b544afba8aee38662138db is 50, key is test_row_0/B:col10/1731975218119/Put/seqid=0 2024-11-19T00:13:39,101 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#A#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:39,101 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/4fd6384343fc44fea12117c0a447322d is 50, key is test_row_0/A:col10/1731975218119/Put/seqid=0 2024-11-19T00:13:39,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741924_1100 (size=13289) 2024-11-19T00:13:39,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741925_1101 (size=13289) 2024-11-19T00:13:39,125 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/4fd6384343fc44fea12117c0a447322d as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/4fd6384343fc44fea12117c0a447322d 2024-11-19T00:13:39,132 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in df30878ae408737da14ed71330503a0a/A of df30878ae408737da14ed71330503a0a into 4fd6384343fc44fea12117c0a447322d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:39,132 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:39,132 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/A, priority=12, startTime=1731975219075; duration=0sec 2024-11-19T00:13:39,132 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:39,132 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:A 2024-11-19T00:13:39,132 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:13:39,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-19T00:13:39,135 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47612 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:13:39,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:39,135 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/C is initiating minor compaction (all files) 2024-11-19T00:13:39,135 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-19T00:13:39,135 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/C in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:39,135 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/62ba1a4802094afaa4b56f5ad70cdc1d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/86353573ced54f75822a14d13c73cd12, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/8b309833728f482aa7cca5a431b3a723, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/9acb79f993a347908341e3e0bf8321ad] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=46.5 K 2024-11-19T00:13:39,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:39,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:39,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:39,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:39,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:39,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:39,136 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62ba1a4802094afaa4b56f5ad70cdc1d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731975215425 2024-11-19T00:13:39,136 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86353573ced54f75822a14d13c73cd12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1731975216264 2024-11-19T00:13:39,137 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b309833728f482aa7cca5a431b3a723, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731975216600 2024-11-19T00:13:39,137 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9acb79f993a347908341e3e0bf8321ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=628, earliestPutTs=1731975218109 2024-11-19T00:13:39,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/ed82a5b45bf445889df920c59e2d0f28 is 50, key is test_row_0/A:col10/1731975218130/Put/seqid=0 2024-11-19T00:13:39,156 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#C#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:39,157 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/6abc673d5bcd4a039f582655713311ff is 50, key is test_row_0/C:col10/1731975218119/Put/seqid=0 2024-11-19T00:13:39,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741926_1102 (size=12301) 2024-11-19T00:13:39,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741927_1103 (size=13289) 2024-11-19T00:13:39,174 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/6abc673d5bcd4a039f582655713311ff as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/6abc673d5bcd4a039f582655713311ff 2024-11-19T00:13:39,182 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in df30878ae408737da14ed71330503a0a/C of df30878ae408737da14ed71330503a0a into 6abc673d5bcd4a039f582655713311ff(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:39,182 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:39,182 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/C, priority=12, startTime=1731975219075; duration=0sec 2024-11-19T00:13:39,183 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:39,183 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:C 2024-11-19T00:13:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on df30878ae408737da14ed71330503a0a 2024-11-19T00:13:39,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:39,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975219293, max timestamp=1731975219355Num uniques 859; 2024-11-19T00:13:39,360 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975219293, max timestamp=1731975219355Num uniques 859; 2024-11-19T00:13:39,360 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:13:39,360 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:13:39,360 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975219293, max timestamp=1731975219355Num uniques 859; 2024-11-19T00:13:39,360 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:13:39,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975279402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:39,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731975219390, max timestamp=1731975219410Num uniques 1068; 2024-11-19T00:13:39,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-19T00:13:39,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731975219390, max timestamp=1731975219410Num uniques 1068; 2024-11-19T00:13:39,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-19T00:13:39,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731975219390, max timestamp=1731975219410Num uniques 1068; 2024-11-19T00:13:39,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-19T00:13:39,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:39,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975279464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:39,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:39,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:39,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975279464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:39,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:39,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975279466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:39,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T00:13:39,516 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/50eeed67f8b544afba8aee38662138db as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/50eeed67f8b544afba8aee38662138db 2024-11-19T00:13:39,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975279514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:39,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:39,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:39,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:39,533 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in df30878ae408737da14ed71330503a0a/B of df30878ae408737da14ed71330503a0a into 50eeed67f8b544afba8aee38662138db(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:39,533 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:39,534 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/B, priority=12, startTime=1731975219075; duration=0sec 2024-11-19T00:13:39,541 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:39,541 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:B 2024-11-19T00:13:39,561 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=638 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/ed82a5b45bf445889df920c59e2d0f28 2024-11-19T00:13:39,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:39,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975279574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:39,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:39,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975279574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:39,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:39,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:39,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:39,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:39,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975279574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:39,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:39,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:39,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:39,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:39,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:39,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:39,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/579ca0be238c43dca43478c430bc4867 is 50, key is test_row_0/B:col10/1731975218130/Put/seqid=0 2024-11-19T00:13:39,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741928_1104 (size=12301) 2024-11-19T00:13:39,605 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=638 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/579ca0be238c43dca43478c430bc4867 2024-11-19T00:13:39,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/a306de4e1c544853a8291ccaf44cf54e is 50, key is test_row_0/C:col10/1731975218130/Put/seqid=0 2024-11-19T00:13:39,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741929_1105 (size=12301) 2024-11-19T00:13:39,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975279734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:39,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:39,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:39,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:39,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975279784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:39,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975279784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:39,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:39,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975279784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:39,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:39,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:39,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:39,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:39,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:39,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:40,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975280043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:40,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:40,054 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=638 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/a306de4e1c544853a8291ccaf44cf54e 2024-11-19T00:13:40,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/ed82a5b45bf445889df920c59e2d0f28 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/ed82a5b45bf445889df920c59e2d0f28 2024-11-19T00:13:40,065 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/ed82a5b45bf445889df920c59e2d0f28, entries=150, sequenceid=638, filesize=12.0 K 2024-11-19T00:13:40,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/579ca0be238c43dca43478c430bc4867 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/579ca0be238c43dca43478c430bc4867 2024-11-19T00:13:40,072 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/579ca0be238c43dca43478c430bc4867, entries=150, sequenceid=638, filesize=12.0 K 2024-11-19T00:13:40,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/a306de4e1c544853a8291ccaf44cf54e as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/a306de4e1c544853a8291ccaf44cf54e 2024-11-19T00:13:40,081 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/a306de4e1c544853a8291ccaf44cf54e, entries=150, sequenceid=638, filesize=12.0 K 2024-11-19T00:13:40,082 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for df30878ae408737da14ed71330503a0a in 947ms, sequenceid=638, compaction requested=false 2024-11-19T00:13:40,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:40,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:40,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-19T00:13:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-19T00:13:40,086 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-19T00:13:40,086 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7170 sec 2024-11-19T00:13:40,088 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.7230 sec 2024-11-19T00:13:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on df30878ae408737da14ed71330503a0a 2024-11-19T00:13:40,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-11-19T00:13:40,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:40,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:13:40,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:40,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:13:40,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:40,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:13:40,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/39a582e2c3874a7c82d85cd15a61e40c is 50, key is test_row_0/A:col10/1731975219439/Put/seqid=0 2024-11-19T00:13:40,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975280107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:40,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975280107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:40,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975280107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:40,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741930_1106 (size=14741) 2024-11-19T00:13:40,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=695 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/39a582e2c3874a7c82d85cd15a61e40c 2024-11-19T00:13:40,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/cf0de4aea36b4befbe9c10706225b3c5 is 50, key is test_row_0/B:col10/1731975219439/Put/seqid=0 2024-11-19T00:13:40,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741931_1107 (size=12301) 2024-11-19T00:13:40,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975280223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:40,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975280223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:40,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975280223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:40,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:40,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975280434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:40,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975280434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:40,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975280434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:40,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:40,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38558 deadline: 1731975280464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:40,467 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_2' in df30878ae408737da14ed71330503a0a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:13:40,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:40,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T00:13:40,514 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:13:40,514 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:13:40,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees 2024-11-19T00:13:40,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T00:13:40,516 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:13:40,517 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:13:40,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:13:40,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=695 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/cf0de4aea36b4befbe9c10706225b3c5 2024-11-19T00:13:40,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/6813d33103b54a6595b5d6fecbffa154 is 50, key is test_row_0/C:col10/1731975219439/Put/seqid=0 2024-11-19T00:13:40,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741932_1108 (size=12301) 2024-11-19T00:13:40,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:40,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38528 deadline: 1731975280554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:40,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T00:13:40,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-19T00:13:40,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:40,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:40,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:40,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:40,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:40,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:40,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38556 deadline: 1731975280744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:40,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:40,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38532 deadline: 1731975280744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:40,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:40,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:40,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1731975280744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:40,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:13:40,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-19T00:13:40,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:40,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:40,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:40,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:40,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:40,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:40,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T00:13:40,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=695 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/6813d33103b54a6595b5d6fecbffa154 2024-11-19T00:13:40,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/39a582e2c3874a7c82d85cd15a61e40c as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/39a582e2c3874a7c82d85cd15a61e40c 2024-11-19T00:13:40,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/39a582e2c3874a7c82d85cd15a61e40c, entries=200, sequenceid=695, filesize=14.4 K 2024-11-19T00:13:40,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/cf0de4aea36b4befbe9c10706225b3c5 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/cf0de4aea36b4befbe9c10706225b3c5 2024-11-19T00:13:40,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/cf0de4aea36b4befbe9c10706225b3c5, entries=150, sequenceid=695, filesize=12.0 K 2024-11-19T00:13:40,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/6813d33103b54a6595b5d6fecbffa154 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/6813d33103b54a6595b5d6fecbffa154 2024-11-19T00:13:40,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/6813d33103b54a6595b5d6fecbffa154, entries=150, sequenceid=695, filesize=12.0 K 2024-11-19T00:13:40,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-19T00:13:40,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:40,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:40,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:40,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:40,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:40,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:40,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.95 KB/430032, currentSize=33.54 KB/34350 for df30878ae408737da14ed71330503a0a in 882ms, sequenceid=695, compaction requested=true 2024-11-19T00:13:40,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:40,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:13:40,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:40,979 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:40,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:13:40,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:40,979 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:40,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df30878ae408737da14ed71330503a0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:13:40,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:40,980 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:40,981 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:40,981 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/A is initiating minor compaction (all files) 2024-11-19T00:13:40,981 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/B is initiating minor compaction (all files) 2024-11-19T00:13:40,981 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/A in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:40,981 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/B in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:40,981 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/50eeed67f8b544afba8aee38662138db, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/579ca0be238c43dca43478c430bc4867, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/cf0de4aea36b4befbe9c10706225b3c5] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=37.0 K 2024-11-19T00:13:40,981 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/4fd6384343fc44fea12117c0a447322d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/ed82a5b45bf445889df920c59e2d0f28, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/39a582e2c3874a7c82d85cd15a61e40c] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=39.4 K 2024-11-19T00:13:40,981 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 50eeed67f8b544afba8aee38662138db, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=628, earliestPutTs=1731975218109 2024-11-19T00:13:40,981 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fd6384343fc44fea12117c0a447322d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=628, earliestPutTs=1731975218109 2024-11-19T00:13:40,982 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 579ca0be238c43dca43478c430bc4867, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=638, earliestPutTs=1731975218127 2024-11-19T00:13:40,982 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed82a5b45bf445889df920c59e2d0f28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=638, earliestPutTs=1731975218127 2024-11-19T00:13:40,982 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting cf0de4aea36b4befbe9c10706225b3c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=695, earliestPutTs=1731975219427 2024-11-19T00:13:40,982 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39a582e2c3874a7c82d85cd15a61e40c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=695, earliestPutTs=1731975219427 2024-11-19T00:13:40,999 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#B#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:40,999 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#A#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:41,000 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/be698bdec85b414ab232aef2e10b0a12 is 50, key is test_row_0/B:col10/1731975219439/Put/seqid=0 2024-11-19T00:13:41,000 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/9343321496ac4bc0b0fb4358aae3a518 is 50, key is test_row_0/A:col10/1731975219439/Put/seqid=0 2024-11-19T00:13:41,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741933_1109 (size=13391) 2024-11-19T00:13:41,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741934_1110 (size=13391) 2024-11-19T00:13:41,017 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/9343321496ac4bc0b0fb4358aae3a518 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/9343321496ac4bc0b0fb4358aae3a518 2024-11-19T00:13:41,023 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/be698bdec85b414ab232aef2e10b0a12 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/be698bdec85b414ab232aef2e10b0a12 2024-11-19T00:13:41,025 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df30878ae408737da14ed71330503a0a/A of df30878ae408737da14ed71330503a0a into 9343321496ac4bc0b0fb4358aae3a518(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:41,025 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:41,025 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/A, priority=13, startTime=1731975220979; duration=0sec 2024-11-19T00:13:41,025 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:41,025 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:A 2024-11-19T00:13:41,026 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:41,028 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:41,028 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): df30878ae408737da14ed71330503a0a/C is initiating minor compaction (all files) 2024-11-19T00:13:41,028 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df30878ae408737da14ed71330503a0a/C in TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:41,029 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/6abc673d5bcd4a039f582655713311ff, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/a306de4e1c544853a8291ccaf44cf54e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/6813d33103b54a6595b5d6fecbffa154] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp, totalSize=37.0 K 2024-11-19T00:13:41,029 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6abc673d5bcd4a039f582655713311ff, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=628, earliestPutTs=1731975218109 2024-11-19T00:13:41,030 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting a306de4e1c544853a8291ccaf44cf54e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=638, earliestPutTs=1731975218127 2024-11-19T00:13:41,030 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6813d33103b54a6595b5d6fecbffa154, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=695, earliestPutTs=1731975219427 2024-11-19T00:13:41,032 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df30878ae408737da14ed71330503a0a/B of df30878ae408737da14ed71330503a0a into be698bdec85b414ab232aef2e10b0a12(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:41,032 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:41,032 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/B, priority=13, startTime=1731975220979; duration=0sec 2024-11-19T00:13:41,032 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:41,032 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:B 2024-11-19T00:13:41,044 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df30878ae408737da14ed71330503a0a#C#compaction#98 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:41,045 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/db6abb8958dd4085bb74f0b98a241503 is 50, key is test_row_0/C:col10/1731975219439/Put/seqid=0 2024-11-19T00:13:41,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741935_1111 (size=13391) 2024-11-19T00:13:41,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-19T00:13:41,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:41,130 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T00:13:41,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:41,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:41,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:41,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:41,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:41,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:41,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/d7664bec3ae24d7cb47671dc4714cb1d is 50, key is test_row_0/A:col10/1731975220105/Put/seqid=0 2024-11-19T00:13:41,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741936_1112 (size=12301) 2024-11-19T00:13:41,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T00:13:41,169 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-19T00:13:41,169 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-19T00:13:41,169 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:13:41,169 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:13:41,170 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-19T00:13:41,170 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:13:41,172 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-19T00:13:41,172 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:13:41,176 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:41,176 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:41,176 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:41,176 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:41,176 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:41,176 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:41,176 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:13:41,176 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:13:41,177 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:41,177 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:13:41,177 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:41,177 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:13:41,255 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-19T00:13:41,255 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:13:41,255 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:41,255 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:41,256 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:13:41,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on df30878ae408737da14ed71330503a0a 2024-11-19T00:13:41,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. as already flushing 2024-11-19T00:13:41,266 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-19T00:13:41,266 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-19T00:13:41,266 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:13:41,266 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:13:41,266 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:41,266 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:41,267 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:41,267 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:41,267 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:13:41,267 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:13:41,458 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/db6abb8958dd4085bb74f0b98a241503 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/db6abb8958dd4085bb74f0b98a241503 2024-11-19T00:13:41,464 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df30878ae408737da14ed71330503a0a/C of df30878ae408737da14ed71330503a0a into db6abb8958dd4085bb74f0b98a241503(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:41,464 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:41,464 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., storeName=df30878ae408737da14ed71330503a0a/C, priority=13, startTime=1731975220979; duration=0sec 2024-11-19T00:13:41,465 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:41,465 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df30878ae408737da14ed71330503a0a:C 2024-11-19T00:13:41,544 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/d7664bec3ae24d7cb47671dc4714cb1d 2024-11-19T00:13:41,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/cbacdd863ea94c5aa444dec974a15c45 is 50, key is test_row_0/B:col10/1731975220105/Put/seqid=0 2024-11-19T00:13:41,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741937_1113 (size=12301) 2024-11-19T00:13:41,560 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/cbacdd863ea94c5aa444dec974a15c45 2024-11-19T00:13:41,566 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-19T00:13:41,566 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:13:41,566 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:41,566 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:41,566 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:13:41,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/429681d2378746d3aeabbd9c2a3b102d is 50, key is test_row_0/C:col10/1731975220105/Put/seqid=0 2024-11-19T00:13:41,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741938_1114 (size=12301) 2024-11-19T00:13:41,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T00:13:41,981 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/429681d2378746d3aeabbd9c2a3b102d 2024-11-19T00:13:41,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/d7664bec3ae24d7cb47671dc4714cb1d as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/d7664bec3ae24d7cb47671dc4714cb1d 2024-11-19T00:13:41,991 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/d7664bec3ae24d7cb47671dc4714cb1d, entries=150, sequenceid=705, filesize=12.0 K 2024-11-19T00:13:41,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/cbacdd863ea94c5aa444dec974a15c45 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/cbacdd863ea94c5aa444dec974a15c45 2024-11-19T00:13:41,997 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/cbacdd863ea94c5aa444dec974a15c45, entries=150, sequenceid=705, filesize=12.0 K 2024-11-19T00:13:41,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/429681d2378746d3aeabbd9c2a3b102d as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/429681d2378746d3aeabbd9c2a3b102d 2024-11-19T00:13:42,003 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/429681d2378746d3aeabbd9c2a3b102d, entries=150, sequenceid=705, filesize=12.0 K 2024-11-19T00:13:42,004 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=26.84 KB/27480 for df30878ae408737da14ed71330503a0a in 874ms, sequenceid=705, compaction requested=false 2024-11-19T00:13:42,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2603): Flush status journal for df30878ae408737da14ed71330503a0a: 2024-11-19T00:13:42,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:42,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-11-19T00:13:42,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-11-19T00:13:42,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-19T00:13:42,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4880 sec 2024-11-19T00:13:42,008 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees in 1.4930 sec 2024-11-19T00:13:42,651 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T00:13:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T00:13:42,674 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:13:44,476 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-19T00:13:44,476 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:13:44,476 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:44,476 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:44,476 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:13:44,476 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T00:13:44,476 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-11-19T00:13:44,476 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 144 2024-11-19T00:13:44,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 117 2024-11-19T00:13:44,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 154 2024-11-19T00:13:44,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-19T00:13:44,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T00:13:44,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2674 2024-11-19T00:13:44,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2672 2024-11-19T00:13:44,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T00:13:44,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1190 2024-11-19T00:13:44,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3567 rows 2024-11-19T00:13:44,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1198 2024-11-19T00:13:44,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3592 rows 2024-11-19T00:13:44,478 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T00:13:44,478 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T00:13:44,478 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:44,478 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:44,478 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:13:44,482 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T00:13:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T00:13:44,492 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975224491"}]},"ts":"1731975224491"} 2024-11-19T00:13:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T00:13:44,494 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T00:13:44,494 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T00:13:44,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T00:13:44,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=df30878ae408737da14ed71330503a0a, UNASSIGN}] 2024-11-19T00:13:44,503 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=df30878ae408737da14ed71330503a0a, UNASSIGN 2024-11-19T00:13:44,505 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=df30878ae408737da14ed71330503a0a, regionState=CLOSING, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:13:44,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=df30878ae408737da14ed71330503a0a, UNASSIGN because future has completed 2024-11-19T00:13:44,507 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T00:13:44,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317}] 2024-11-19T00:13:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T00:13:44,667 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close df30878ae408737da14ed71330503a0a 2024-11-19T00:13:44,668 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T00:13:44,669 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing df30878ae408737da14ed71330503a0a, disabling compactions & flushes 2024-11-19T00:13:44,669 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:44,669 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:44,670 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. after waiting 0 ms 2024-11-19T00:13:44,670 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:44,670 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(2902): Flushing df30878ae408737da14ed71330503a0a 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T00:13:44,671 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=A 2024-11-19T00:13:44,671 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:44,671 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=B 2024-11-19T00:13:44,671 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:44,671 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactingMemStore(205): FLUSHING TO DISK df30878ae408737da14ed71330503a0a, store=C 2024-11-19T00:13:44,671 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:44,677 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/992f822643d04d76bb9e39cf62e216fa is 50, key is test_row_0/A:col10/1731975221564/Put/seqid=0 2024-11-19T00:13:44,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741939_1115 (size=12301) 2024-11-19T00:13:44,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T00:13:45,083 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/992f822643d04d76bb9e39cf62e216fa 2024-11-19T00:13:45,096 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/f1803c52fd394b88b1b0b6b465d41662 is 50, key is test_row_0/B:col10/1731975221564/Put/seqid=0 2024-11-19T00:13:45,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741940_1116 (size=12301) 2024-11-19T00:13:45,108 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/f1803c52fd394b88b1b0b6b465d41662 2024-11-19T00:13:45,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T00:13:45,129 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/3fef44fa0afa45399c1cb497fae3d454 is 50, key is test_row_0/C:col10/1731975221564/Put/seqid=0 2024-11-19T00:13:45,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741941_1117 (size=12301) 2024-11-19T00:13:45,534 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/3fef44fa0afa45399c1cb497fae3d454 2024-11-19T00:13:45,540 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/A/992f822643d04d76bb9e39cf62e216fa as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/992f822643d04d76bb9e39cf62e216fa 2024-11-19T00:13:45,545 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/992f822643d04d76bb9e39cf62e216fa, entries=150, sequenceid=714, filesize=12.0 K 2024-11-19T00:13:45,546 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/B/f1803c52fd394b88b1b0b6b465d41662 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/f1803c52fd394b88b1b0b6b465d41662 2024-11-19T00:13:45,551 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/f1803c52fd394b88b1b0b6b465d41662, entries=150, sequenceid=714, filesize=12.0 K 2024-11-19T00:13:45,553 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/.tmp/C/3fef44fa0afa45399c1cb497fae3d454 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/3fef44fa0afa45399c1cb497fae3d454 2024-11-19T00:13:45,558 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/3fef44fa0afa45399c1cb497fae3d454, entries=150, sequenceid=714, filesize=12.0 K 2024-11-19T00:13:45,559 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for df30878ae408737da14ed71330503a0a in 889ms, sequenceid=714, compaction requested=true 2024-11-19T00:13:45,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/c90ea3b073f844e6b908f37f5b3a3db8, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/66fc606e515e47a9a071849e405f9aea, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/12d77166d314435fa88359e10fdd1f0c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/f125f13092f446e0a12ef0cf43648cfb, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/6d47bb477710409a891bb44cf3c68187, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/c2a7411acb1b453ab469b5038d558b5c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/c4cb4f16b92d4c08a1e5ed512860db1c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/cc26c3f26fac422089c0e09382cc8c07, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/597cad2e0c8c49d8a2c2093124250e25, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/4941ac08ecf443baa822f87797129587, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/84c133b233774befa09b0e58f8f18437, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/b7b7af5cef3b4a3e8f8cdeb980a06205, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/e2665cf6a37a42ebb537184ca27f7bd4, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/30b65ad1b841437a81b1547ec91c36c4, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/20648145c6f645fdb73d2819597562a0, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/d7066e5d425946edb882f10a47b51ff7, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/806478de29444313a47445e568f8ed30, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/5a714035d6bd4c02b0ac2de72005bb4b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/3b974520699a4bde9820b9bd0a76183c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/62fda2e1f7ac4864a23d69e33448db33, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/681f08ca588b455bb791c074621ac5f4, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/f4c4db175c7a481eb280758a368a042d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/b462be64fe3e4a07bc2fcd8c7c6933db, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/2ecc1d78faf2423ea8372bffd92e2358, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/6fcbcf84028744019dfaf1339b34f419, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/65d76253ef284f0b93a4fca32edd604a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/2482d86d1f844fbaa6150686f154839c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/d5a01195b5cb4051a6917c66dc2c934a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/91991e3efadf4fff88a8d5644bcf5705, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/4fd6384343fc44fea12117c0a447322d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/ed82a5b45bf445889df920c59e2d0f28, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/39a582e2c3874a7c82d85cd15a61e40c] to archive 2024-11-19T00:13:45,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T00:13:45,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/c90ea3b073f844e6b908f37f5b3a3db8 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/c90ea3b073f844e6b908f37f5b3a3db8 2024-11-19T00:13:45,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/66fc606e515e47a9a071849e405f9aea to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/66fc606e515e47a9a071849e405f9aea 2024-11-19T00:13:45,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/12d77166d314435fa88359e10fdd1f0c to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/12d77166d314435fa88359e10fdd1f0c 2024-11-19T00:13:45,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/f125f13092f446e0a12ef0cf43648cfb to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/f125f13092f446e0a12ef0cf43648cfb 2024-11-19T00:13:45,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/6d47bb477710409a891bb44cf3c68187 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/6d47bb477710409a891bb44cf3c68187 2024-11-19T00:13:45,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/c2a7411acb1b453ab469b5038d558b5c to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/c2a7411acb1b453ab469b5038d558b5c 2024-11-19T00:13:45,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/c4cb4f16b92d4c08a1e5ed512860db1c to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/c4cb4f16b92d4c08a1e5ed512860db1c 2024-11-19T00:13:45,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/cc26c3f26fac422089c0e09382cc8c07 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/cc26c3f26fac422089c0e09382cc8c07 2024-11-19T00:13:45,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/597cad2e0c8c49d8a2c2093124250e25 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/597cad2e0c8c49d8a2c2093124250e25 2024-11-19T00:13:45,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/4941ac08ecf443baa822f87797129587 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/4941ac08ecf443baa822f87797129587 2024-11-19T00:13:45,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/84c133b233774befa09b0e58f8f18437 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/84c133b233774befa09b0e58f8f18437 2024-11-19T00:13:45,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/b7b7af5cef3b4a3e8f8cdeb980a06205 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/b7b7af5cef3b4a3e8f8cdeb980a06205 2024-11-19T00:13:45,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/e2665cf6a37a42ebb537184ca27f7bd4 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/e2665cf6a37a42ebb537184ca27f7bd4 2024-11-19T00:13:45,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/30b65ad1b841437a81b1547ec91c36c4 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/30b65ad1b841437a81b1547ec91c36c4 2024-11-19T00:13:45,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/20648145c6f645fdb73d2819597562a0 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/20648145c6f645fdb73d2819597562a0 2024-11-19T00:13:45,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/d7066e5d425946edb882f10a47b51ff7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/d7066e5d425946edb882f10a47b51ff7 2024-11-19T00:13:45,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/806478de29444313a47445e568f8ed30 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/806478de29444313a47445e568f8ed30 2024-11-19T00:13:45,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/5a714035d6bd4c02b0ac2de72005bb4b to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/5a714035d6bd4c02b0ac2de72005bb4b 2024-11-19T00:13:45,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/3b974520699a4bde9820b9bd0a76183c to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/3b974520699a4bde9820b9bd0a76183c 2024-11-19T00:13:45,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/62fda2e1f7ac4864a23d69e33448db33 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/62fda2e1f7ac4864a23d69e33448db33 2024-11-19T00:13:45,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/681f08ca588b455bb791c074621ac5f4 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/681f08ca588b455bb791c074621ac5f4 2024-11-19T00:13:45,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/f4c4db175c7a481eb280758a368a042d to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/f4c4db175c7a481eb280758a368a042d 2024-11-19T00:13:45,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/b462be64fe3e4a07bc2fcd8c7c6933db to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/b462be64fe3e4a07bc2fcd8c7c6933db 2024-11-19T00:13:45,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/2ecc1d78faf2423ea8372bffd92e2358 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/2ecc1d78faf2423ea8372bffd92e2358 2024-11-19T00:13:45,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/6fcbcf84028744019dfaf1339b34f419 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/6fcbcf84028744019dfaf1339b34f419 2024-11-19T00:13:45,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/65d76253ef284f0b93a4fca32edd604a to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/65d76253ef284f0b93a4fca32edd604a 2024-11-19T00:13:45,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/2482d86d1f844fbaa6150686f154839c to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/2482d86d1f844fbaa6150686f154839c 2024-11-19T00:13:45,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/d5a01195b5cb4051a6917c66dc2c934a to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/d5a01195b5cb4051a6917c66dc2c934a 2024-11-19T00:13:45,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/91991e3efadf4fff88a8d5644bcf5705 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/91991e3efadf4fff88a8d5644bcf5705 2024-11-19T00:13:45,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/4fd6384343fc44fea12117c0a447322d to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/4fd6384343fc44fea12117c0a447322d 2024-11-19T00:13:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T00:13:45,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/ed82a5b45bf445889df920c59e2d0f28 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/ed82a5b45bf445889df920c59e2d0f28 2024-11-19T00:13:45,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/39a582e2c3874a7c82d85cd15a61e40c to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/39a582e2c3874a7c82d85cd15a61e40c 2024-11-19T00:13:45,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/d289013c3d55418aae749e7e77a4a6f8, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/8ad2725a53f74e17ad6df02f587ab850, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/cb3f022673424792ab1d29199dca6463, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/e62d87fd768a44efb70074b77b86338f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/673e9b899b3e4a1e8665ad26f2e25adb, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/16f4a9668cc24f7d86dafd8ceab20c03, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/5e8fd30446354b3da5b89dd6d501ee66, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/c1444bc6675640a7be0c8c3a92670828, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/0ed291fa135440488e9170294cb7d760, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/4fde3ec3efcd4b118b23c9a83aa91952, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/b2203caf1fab43b797828b414c62fbe9, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/6154f5d0b359421f8137517913f45847, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/5afbe921fddc40cab85e14aa88283251, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/9b2861fa1e254defa7138d70fd97c6bc, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/db31719e72fb4d18ac7ec0295612e6a6, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/698d110b45d04a61b7430dc875df8647, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/ba7972f80d474018a278c57689a88844, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/919a43ae3b89464c8c97ab87fa2519b6, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/711dae44ef9d4240a2bbe3121d2f79a8, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/c6a24eda17f2411094d31835e6e9e576, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/de2c5bd856474bb1a0e96ff658779654, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/e01d4ed838864ac59add774c7a6b37b4, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/f300d8eb38cd4c8f9ecd7deb3048b413, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/b8107d8391864d2189271e2affa876f4, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/fbe35e3089cc48a78c9b25f533dc9300, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/8df1d511f5ab4359a22b7101867b0684, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/1da4a672ae9b410bae2f06325ebde359, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/4dde8e6400e9417f8006b450210507ac, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/50eeed67f8b544afba8aee38662138db, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/84a53c525d5949a7a41c9be50e24c0dc, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/579ca0be238c43dca43478c430bc4867, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/cf0de4aea36b4befbe9c10706225b3c5] to archive 2024-11-19T00:13:45,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T00:13:45,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/d289013c3d55418aae749e7e77a4a6f8 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/d289013c3d55418aae749e7e77a4a6f8 2024-11-19T00:13:45,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/8ad2725a53f74e17ad6df02f587ab850 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/8ad2725a53f74e17ad6df02f587ab850 2024-11-19T00:13:45,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/cb3f022673424792ab1d29199dca6463 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/cb3f022673424792ab1d29199dca6463 2024-11-19T00:13:45,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/e62d87fd768a44efb70074b77b86338f to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/e62d87fd768a44efb70074b77b86338f 2024-11-19T00:13:45,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/673e9b899b3e4a1e8665ad26f2e25adb to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/673e9b899b3e4a1e8665ad26f2e25adb 2024-11-19T00:13:45,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/16f4a9668cc24f7d86dafd8ceab20c03 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/16f4a9668cc24f7d86dafd8ceab20c03 2024-11-19T00:13:45,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/5e8fd30446354b3da5b89dd6d501ee66 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/5e8fd30446354b3da5b89dd6d501ee66 2024-11-19T00:13:45,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/c1444bc6675640a7be0c8c3a92670828 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/c1444bc6675640a7be0c8c3a92670828 2024-11-19T00:13:45,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/0ed291fa135440488e9170294cb7d760 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/0ed291fa135440488e9170294cb7d760 2024-11-19T00:13:45,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/4fde3ec3efcd4b118b23c9a83aa91952 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/4fde3ec3efcd4b118b23c9a83aa91952 2024-11-19T00:13:45,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/b2203caf1fab43b797828b414c62fbe9 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/b2203caf1fab43b797828b414c62fbe9 2024-11-19T00:13:45,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/6154f5d0b359421f8137517913f45847 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/6154f5d0b359421f8137517913f45847 2024-11-19T00:13:45,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/5afbe921fddc40cab85e14aa88283251 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/5afbe921fddc40cab85e14aa88283251 2024-11-19T00:13:45,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/9b2861fa1e254defa7138d70fd97c6bc to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/9b2861fa1e254defa7138d70fd97c6bc 2024-11-19T00:13:45,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/db31719e72fb4d18ac7ec0295612e6a6 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/db31719e72fb4d18ac7ec0295612e6a6 2024-11-19T00:13:45,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/698d110b45d04a61b7430dc875df8647 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/698d110b45d04a61b7430dc875df8647 2024-11-19T00:13:45,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/ba7972f80d474018a278c57689a88844 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/ba7972f80d474018a278c57689a88844 2024-11-19T00:13:45,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/919a43ae3b89464c8c97ab87fa2519b6 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/919a43ae3b89464c8c97ab87fa2519b6 2024-11-19T00:13:45,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/711dae44ef9d4240a2bbe3121d2f79a8 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/711dae44ef9d4240a2bbe3121d2f79a8 2024-11-19T00:13:45,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/c6a24eda17f2411094d31835e6e9e576 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/c6a24eda17f2411094d31835e6e9e576 2024-11-19T00:13:45,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/de2c5bd856474bb1a0e96ff658779654 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/de2c5bd856474bb1a0e96ff658779654 2024-11-19T00:13:45,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/e01d4ed838864ac59add774c7a6b37b4 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/e01d4ed838864ac59add774c7a6b37b4 2024-11-19T00:13:45,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/f300d8eb38cd4c8f9ecd7deb3048b413 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/f300d8eb38cd4c8f9ecd7deb3048b413 2024-11-19T00:13:45,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/b8107d8391864d2189271e2affa876f4 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/b8107d8391864d2189271e2affa876f4 2024-11-19T00:13:45,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/fbe35e3089cc48a78c9b25f533dc9300 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/fbe35e3089cc48a78c9b25f533dc9300 2024-11-19T00:13:45,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/8df1d511f5ab4359a22b7101867b0684 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/8df1d511f5ab4359a22b7101867b0684 2024-11-19T00:13:45,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/1da4a672ae9b410bae2f06325ebde359 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/1da4a672ae9b410bae2f06325ebde359 2024-11-19T00:13:45,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/4dde8e6400e9417f8006b450210507ac to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/4dde8e6400e9417f8006b450210507ac 2024-11-19T00:13:45,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/50eeed67f8b544afba8aee38662138db to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/50eeed67f8b544afba8aee38662138db 2024-11-19T00:13:45,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/84a53c525d5949a7a41c9be50e24c0dc to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/84a53c525d5949a7a41c9be50e24c0dc 2024-11-19T00:13:45,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/579ca0be238c43dca43478c430bc4867 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/579ca0be238c43dca43478c430bc4867 2024-11-19T00:13:45,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/cf0de4aea36b4befbe9c10706225b3c5 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/cf0de4aea36b4befbe9c10706225b3c5 2024-11-19T00:13:45,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/039566c1a4084b0d9f662f8a23a9ffbe, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/3b4f3e8c43a946329842ddd3030b79c0, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/b77013f572824aff84678aa02670dfe4, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/506912e6030546f5a80dd4b1e50366e3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/5d3856aa7a644f6ea4459d1a7b798030, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/a048092119804410a5398c1f24e02551, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/522f73e8f23a483a8b207fb0651be6c0, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/771ef694a677497da4b0730f938730a5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/526fd821e9de44eab94a6e17fd68fc72, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/7237c46aa0374f48bc244373ef268421, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/7421f66d2ec1498b89b5982469336fdb, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/228b39eef60645c9a06b090418fae81a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/a83b5b44f6e5432791395af0de664f38, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/2d3a367f406b4c6ea5ff040c16cc36ad, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/db56c89a2988472591b61c1d4b7a5a49, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/5d63f79b38fa42fbb6efb8da1cdeb1b2, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/15b87fb4d8514f93b0e85aa93c83c389, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/db42e3066f704e0f9dbc4974ca450dce, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/b6ff8e9e485d4acdb2b45a6b65452fe6, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/6be1c1d52fe84884ba17b6461798ef08, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/1b2a1e8f1e9b49e38ddd0fdb9ea5ee47, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/ad58a0031e5c488aba139ce30c083c94, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/54788d241c3f4540ae45941418745b14, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/76ebcf67772e4316b89d791f6c63bd4d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/62ba1a4802094afaa4b56f5ad70cdc1d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/3b091b0834c34fb9bd9f99b1c4133ee6, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/86353573ced54f75822a14d13c73cd12, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/8b309833728f482aa7cca5a431b3a723, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/6abc673d5bcd4a039f582655713311ff, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/9acb79f993a347908341e3e0bf8321ad, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/a306de4e1c544853a8291ccaf44cf54e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/6813d33103b54a6595b5d6fecbffa154] to archive 2024-11-19T00:13:45,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T00:13:45,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/039566c1a4084b0d9f662f8a23a9ffbe to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/039566c1a4084b0d9f662f8a23a9ffbe 2024-11-19T00:13:45,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/3b4f3e8c43a946329842ddd3030b79c0 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/3b4f3e8c43a946329842ddd3030b79c0 2024-11-19T00:13:45,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/b77013f572824aff84678aa02670dfe4 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/b77013f572824aff84678aa02670dfe4 2024-11-19T00:13:45,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/506912e6030546f5a80dd4b1e50366e3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/506912e6030546f5a80dd4b1e50366e3 2024-11-19T00:13:45,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/5d3856aa7a644f6ea4459d1a7b798030 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/5d3856aa7a644f6ea4459d1a7b798030 2024-11-19T00:13:45,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/a048092119804410a5398c1f24e02551 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/a048092119804410a5398c1f24e02551 2024-11-19T00:13:45,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/522f73e8f23a483a8b207fb0651be6c0 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/522f73e8f23a483a8b207fb0651be6c0 2024-11-19T00:13:45,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/771ef694a677497da4b0730f938730a5 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/771ef694a677497da4b0730f938730a5 2024-11-19T00:13:45,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/526fd821e9de44eab94a6e17fd68fc72 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/526fd821e9de44eab94a6e17fd68fc72 2024-11-19T00:13:45,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/7237c46aa0374f48bc244373ef268421 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/7237c46aa0374f48bc244373ef268421 2024-11-19T00:13:45,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/7421f66d2ec1498b89b5982469336fdb to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/7421f66d2ec1498b89b5982469336fdb 2024-11-19T00:13:45,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/228b39eef60645c9a06b090418fae81a to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/228b39eef60645c9a06b090418fae81a 2024-11-19T00:13:45,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/a83b5b44f6e5432791395af0de664f38 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/a83b5b44f6e5432791395af0de664f38 2024-11-19T00:13:45,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/2d3a367f406b4c6ea5ff040c16cc36ad to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/2d3a367f406b4c6ea5ff040c16cc36ad 2024-11-19T00:13:45,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/db56c89a2988472591b61c1d4b7a5a49 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/db56c89a2988472591b61c1d4b7a5a49 2024-11-19T00:13:45,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/5d63f79b38fa42fbb6efb8da1cdeb1b2 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/5d63f79b38fa42fbb6efb8da1cdeb1b2 2024-11-19T00:13:45,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/15b87fb4d8514f93b0e85aa93c83c389 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/15b87fb4d8514f93b0e85aa93c83c389 2024-11-19T00:13:45,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/db42e3066f704e0f9dbc4974ca450dce to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/db42e3066f704e0f9dbc4974ca450dce 2024-11-19T00:13:45,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/b6ff8e9e485d4acdb2b45a6b65452fe6 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/b6ff8e9e485d4acdb2b45a6b65452fe6 2024-11-19T00:13:45,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/6be1c1d52fe84884ba17b6461798ef08 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/6be1c1d52fe84884ba17b6461798ef08 2024-11-19T00:13:45,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/1b2a1e8f1e9b49e38ddd0fdb9ea5ee47 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/1b2a1e8f1e9b49e38ddd0fdb9ea5ee47 2024-11-19T00:13:45,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/ad58a0031e5c488aba139ce30c083c94 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/ad58a0031e5c488aba139ce30c083c94 2024-11-19T00:13:45,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/54788d241c3f4540ae45941418745b14 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/54788d241c3f4540ae45941418745b14 2024-11-19T00:13:45,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/76ebcf67772e4316b89d791f6c63bd4d to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/76ebcf67772e4316b89d791f6c63bd4d 2024-11-19T00:13:45,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/62ba1a4802094afaa4b56f5ad70cdc1d to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/62ba1a4802094afaa4b56f5ad70cdc1d 2024-11-19T00:13:45,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/3b091b0834c34fb9bd9f99b1c4133ee6 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/3b091b0834c34fb9bd9f99b1c4133ee6 2024-11-19T00:13:45,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/86353573ced54f75822a14d13c73cd12 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/86353573ced54f75822a14d13c73cd12 2024-11-19T00:13:45,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/8b309833728f482aa7cca5a431b3a723 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/8b309833728f482aa7cca5a431b3a723 2024-11-19T00:13:45,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/6abc673d5bcd4a039f582655713311ff to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/6abc673d5bcd4a039f582655713311ff 2024-11-19T00:13:45,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/9acb79f993a347908341e3e0bf8321ad to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/9acb79f993a347908341e3e0bf8321ad 2024-11-19T00:13:45,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/a306de4e1c544853a8291ccaf44cf54e to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/a306de4e1c544853a8291ccaf44cf54e 2024-11-19T00:13:45,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/6813d33103b54a6595b5d6fecbffa154 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/6813d33103b54a6595b5d6fecbffa154 2024-11-19T00:13:45,804 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/recovered.edits/717.seqid, newMaxSeqId=717, maxSeqId=1 2024-11-19T00:13:45,805 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. 2024-11-19T00:13:45,805 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for df30878ae408737da14ed71330503a0a: Waiting for close lock at 1731975224669Running coprocessor pre-close hooks at 1731975224669Disabling compacts and flushes for region at 1731975224669Disabling writes for close at 1731975224670 (+1 ms)Obtaining lock to block concurrent updates at 1731975224670Preparing flush snapshotting stores in df30878ae408737da14ed71330503a0a at 1731975224670Finished memstore snapshotting TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731975224672 (+2 ms)Flushing stores of TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a. at 1731975224673 (+1 ms)Flushing df30878ae408737da14ed71330503a0a/A: creating writer at 1731975224673Flushing df30878ae408737da14ed71330503a0a/A: appending metadata at 1731975224677 (+4 ms)Flushing df30878ae408737da14ed71330503a0a/A: closing flushed file at 1731975224677Flushing df30878ae408737da14ed71330503a0a/B: creating writer at 1731975225089 (+412 ms)Flushing df30878ae408737da14ed71330503a0a/B: appending metadata at 1731975225095 (+6 ms)Flushing df30878ae408737da14ed71330503a0a/B: closing flushed file at 1731975225095Flushing df30878ae408737da14ed71330503a0a/C: creating writer at 1731975225112 (+17 ms)Flushing df30878ae408737da14ed71330503a0a/C: appending metadata at 1731975225128 (+16 ms)Flushing df30878ae408737da14ed71330503a0a/C: closing flushed file at 1731975225128Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4448f585: reopening flushed file at 1731975225539 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b9a5b64: reopening flushed file at 1731975225545 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@168740e6: reopening flushed file at 1731975225552 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for df30878ae408737da14ed71330503a0a in 889ms, sequenceid=714, compaction requested=true at 1731975225559 (+7 ms)Writing region close event to WAL at 1731975225797 (+238 ms)Running coprocessor post-close hooks at 1731975225805 (+8 ms)Closed at 1731975225805 2024-11-19T00:13:45,809 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed df30878ae408737da14ed71330503a0a 2024-11-19T00:13:45,809 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=df30878ae408737da14ed71330503a0a, regionState=CLOSED 2024-11-19T00:13:45,812 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 because future has completed 2024-11-19T00:13:45,817 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-19T00:13:45,817 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure df30878ae408737da14ed71330503a0a, server=6b1531871d35,38567,1731975194317 in 1.3060 sec 2024-11-19T00:13:45,819 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-19T00:13:45,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=df30878ae408737da14ed71330503a0a, UNASSIGN in 1.3150 sec 2024-11-19T00:13:45,825 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-19T00:13:45,825 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.3250 sec 2024-11-19T00:13:45,827 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975225827"}]},"ts":"1731975225827"} 2024-11-19T00:13:45,830 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T00:13:45,830 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T00:13:45,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.3460 sec 2024-11-19T00:13:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T00:13:46,634 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:13:46,637 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T00:13:46,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:13:46,643 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:13:46,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-19T00:13:46,645 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=35, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:13:46,651 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a 2024-11-19T00:13:46,655 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A, FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B, FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C, FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/recovered.edits] 2024-11-19T00:13:46,659 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/9343321496ac4bc0b0fb4358aae3a518 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/9343321496ac4bc0b0fb4358aae3a518 2024-11-19T00:13:46,660 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/992f822643d04d76bb9e39cf62e216fa to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/992f822643d04d76bb9e39cf62e216fa 2024-11-19T00:13:46,662 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/d7664bec3ae24d7cb47671dc4714cb1d to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/A/d7664bec3ae24d7cb47671dc4714cb1d 2024-11-19T00:13:46,665 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/be698bdec85b414ab232aef2e10b0a12 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/be698bdec85b414ab232aef2e10b0a12 2024-11-19T00:13:46,667 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/cbacdd863ea94c5aa444dec974a15c45 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/cbacdd863ea94c5aa444dec974a15c45 2024-11-19T00:13:46,668 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/f1803c52fd394b88b1b0b6b465d41662 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/B/f1803c52fd394b88b1b0b6b465d41662 2024-11-19T00:13:46,671 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/3fef44fa0afa45399c1cb497fae3d454 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/3fef44fa0afa45399c1cb497fae3d454 2024-11-19T00:13:46,673 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/429681d2378746d3aeabbd9c2a3b102d to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/429681d2378746d3aeabbd9c2a3b102d 2024-11-19T00:13:46,674 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/db6abb8958dd4085bb74f0b98a241503 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/C/db6abb8958dd4085bb74f0b98a241503 2024-11-19T00:13:46,677 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/recovered.edits/717.seqid to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a/recovered.edits/717.seqid 2024-11-19T00:13:46,678 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/df30878ae408737da14ed71330503a0a 2024-11-19T00:13:46,678 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T00:13:46,683 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=35, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:13:46,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-19T00:13:46,696 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T00:13:46,745 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T00:13:46,749 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=35, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:13:46,750 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T00:13:46,750 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731975226750"}]},"ts":"9223372036854775807"} 2024-11-19T00:13:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-19T00:13:46,758 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T00:13:46,758 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => df30878ae408737da14ed71330503a0a, NAME => 'TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T00:13:46,758 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T00:13:46,758 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731975226758"}]},"ts":"9223372036854775807"} 2024-11-19T00:13:46,763 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T00:13:46,764 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=35, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:13:46,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 126 msec 2024-11-19T00:13:46,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-19T00:13:46,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T00:13:46,966 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:13:46,985 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_625823604_22 at /127.0.0.1:34040 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/6b1531871d35:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;6b1531871d35:38567-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=367 (was 231) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2659 (was 2432) - AvailableMemoryMB LEAK? - 2024-11-19T00:13:47,000 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=367, ProcessCount=11, AvailableMemoryMB=2657 2024-11-19T00:13:47,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T00:13:47,002 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T00:13:47,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T00:13:47,005 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T00:13:47,005 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:47,005 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 36 2024-11-19T00:13:47,006 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T00:13:47,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T00:13:47,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741942_1118 (size=1061) 2024-11-19T00:13:47,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T00:13:47,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T00:13:47,418 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2 2024-11-19T00:13:47,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741943_1119 (size=53) 2024-11-19T00:13:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T00:13:47,825 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:13:47,825 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0f7b113264f1c67c501d9bd1480cc2b7, disabling compactions & flushes 2024-11-19T00:13:47,825 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:47,825 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:47,825 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. after waiting 0 ms 2024-11-19T00:13:47,825 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:47,825 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:47,825 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0f7b113264f1c67c501d9bd1480cc2b7: Waiting for close lock at 1731975227825Disabling compacts and flushes for region at 1731975227825Disabling writes for close at 1731975227825Writing region close event to WAL at 1731975227825Closed at 1731975227825 2024-11-19T00:13:47,827 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T00:13:47,827 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731975227827"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731975227827"}]},"ts":"1731975227827"} 2024-11-19T00:13:47,829 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T00:13:47,830 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T00:13:47,831 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975227830"}]},"ts":"1731975227830"} 2024-11-19T00:13:47,832 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T00:13:47,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f7b113264f1c67c501d9bd1480cc2b7, ASSIGN}] 2024-11-19T00:13:47,834 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f7b113264f1c67c501d9bd1480cc2b7, ASSIGN 2024-11-19T00:13:47,835 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f7b113264f1c67c501d9bd1480cc2b7, ASSIGN; state=OFFLINE, location=6b1531871d35,38567,1731975194317; forceNewPlan=false, retain=false 2024-11-19T00:13:47,986 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=0f7b113264f1c67c501d9bd1480cc2b7, regionState=OPENING, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:13:47,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f7b113264f1c67c501d9bd1480cc2b7, ASSIGN because future has completed 2024-11-19T00:13:47,989 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317}] 2024-11-19T00:13:48,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T00:13:48,149 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:48,149 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} 2024-11-19T00:13:48,150 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:48,150 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:13:48,150 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:48,150 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:48,152 INFO [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:48,154 INFO [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:13:48,155 INFO [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f7b113264f1c67c501d9bd1480cc2b7 columnFamilyName A 2024-11-19T00:13:48,156 DEBUG [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:48,156 INFO [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] regionserver.HStore(327): Store=0f7b113264f1c67c501d9bd1480cc2b7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:13:48,157 INFO [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:48,158 INFO [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:13:48,159 INFO [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f7b113264f1c67c501d9bd1480cc2b7 columnFamilyName B 2024-11-19T00:13:48,159 DEBUG [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:48,160 INFO [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] regionserver.HStore(327): Store=0f7b113264f1c67c501d9bd1480cc2b7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:13:48,160 INFO [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:48,161 INFO [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:13:48,162 INFO [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f7b113264f1c67c501d9bd1480cc2b7 columnFamilyName C 2024-11-19T00:13:48,162 DEBUG [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:48,163 INFO [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] regionserver.HStore(327): Store=0f7b113264f1c67c501d9bd1480cc2b7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:13:48,163 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:48,163 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:48,164 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:48,165 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:48,166 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:48,166 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:48,167 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T00:13:48,168 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:48,173 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T00:13:48,174 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 0f7b113264f1c67c501d9bd1480cc2b7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63542319, jitterRate=-0.05314566195011139}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T00:13:48,174 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:48,175 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 0f7b113264f1c67c501d9bd1480cc2b7: Running coprocessor pre-open hook at 1731975228150Writing region info on filesystem at 1731975228150Initializing all the Stores at 1731975228152 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975228152Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975228152Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975228152Cleaning up temporary data from old regions at 1731975228166 (+14 ms)Running coprocessor post-open hooks at 1731975228174 (+8 ms)Region opened successfully at 1731975228175 (+1 ms) 2024-11-19T00:13:48,177 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., pid=38, masterSystemTime=1731975228142 2024-11-19T00:13:48,185 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:48,185 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:48,186 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=0f7b113264f1c67c501d9bd1480cc2b7, regionState=OPEN, openSeqNum=2, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:13:48,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 because future has completed 2024-11-19T00:13:48,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-19T00:13:48,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 in 202 msec 2024-11-19T00:13:48,198 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-19T00:13:48,198 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f7b113264f1c67c501d9bd1480cc2b7, ASSIGN in 361 msec 2024-11-19T00:13:48,199 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T00:13:48,200 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975228200"}]},"ts":"1731975228200"} 2024-11-19T00:13:48,202 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T00:13:48,204 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T00:13:48,207 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2010 sec 2024-11-19T00:13:49,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T00:13:49,155 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:13:49,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:49,156 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:13:49,157 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:13:49,159 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:13:49,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:13:49,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:13:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64d49718, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:13:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:13:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:49,162 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:13:49,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@121de19e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:49,163 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:13:49,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:13:49,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:13:49,167 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:13:49,169 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T00:13:49,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 6b1531871d35,33843,1731975193570 2024-11-19T00:13:49,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@c90b79a 2024-11-19T00:13:49,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T00:13:49,172 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T00:13:49,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T00:13:49,179 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T00:13:49,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T00:13:49,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741944_1120 (size=1097) 2024-11-19T00:13:49,607 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-19T00:13:49,607 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-19T00:13:49,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T00:13:49,621 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f7b113264f1c67c501d9bd1480cc2b7, REOPEN/MOVE}] 2024-11-19T00:13:49,621 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f7b113264f1c67c501d9bd1480cc2b7, REOPEN/MOVE 2024-11-19T00:13:49,622 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=0f7b113264f1c67c501d9bd1480cc2b7, regionState=CLOSING, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:13:49,624 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f7b113264f1c67c501d9bd1480cc2b7, REOPEN/MOVE because future has completed 2024-11-19T00:13:49,624 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T00:13:49,625 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317}] 2024-11-19T00:13:49,777 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(122): Close 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:49,778 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T00:13:49,778 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1722): Closing 0f7b113264f1c67c501d9bd1480cc2b7, disabling compactions & flushes 2024-11-19T00:13:49,778 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:49,778 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:49,778 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. after waiting 0 ms 2024-11-19T00:13:49,778 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:49,783 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T00:13:49,784 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:49,784 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1676): Region close journal for 0f7b113264f1c67c501d9bd1480cc2b7: Waiting for close lock at 1731975229778Running coprocessor pre-close hooks at 1731975229778Disabling compacts and flushes for region at 1731975229778Disabling writes for close at 1731975229778Writing region close event to WAL at 1731975229778Running coprocessor post-close hooks at 1731975229784 (+6 ms)Closed at 1731975229784 2024-11-19T00:13:49,784 WARN [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegionServer(3299): Not adding moved region record: 0f7b113264f1c67c501d9bd1480cc2b7 to self. 2024-11-19T00:13:49,787 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(157): Closed 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:49,788 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=0f7b113264f1c67c501d9bd1480cc2b7, regionState=CLOSED 2024-11-19T00:13:49,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 because future has completed 2024-11-19T00:13:49,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=41 2024-11-19T00:13:49,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=41, state=SUCCESS, hasLock=false; CloseRegionProcedure 0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 in 168 msec 2024-11-19T00:13:49,796 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f7b113264f1c67c501d9bd1480cc2b7, REOPEN/MOVE; state=CLOSED, location=6b1531871d35,38567,1731975194317; forceNewPlan=false, retain=true 2024-11-19T00:13:49,947 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=0f7b113264f1c67c501d9bd1480cc2b7, regionState=OPENING, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:13:49,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f7b113264f1c67c501d9bd1480cc2b7, REOPEN/MOVE because future has completed 2024-11-19T00:13:49,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317}] 2024-11-19T00:13:50,105 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:50,106 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} 2024-11-19T00:13:50,106 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:50,106 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:13:50,106 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:50,106 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:50,109 INFO [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:50,110 INFO [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:13:50,115 INFO [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f7b113264f1c67c501d9bd1480cc2b7 columnFamilyName A 2024-11-19T00:13:50,117 DEBUG [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:50,117 INFO [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] regionserver.HStore(327): Store=0f7b113264f1c67c501d9bd1480cc2b7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:13:50,118 INFO [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:50,119 INFO [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:13:50,119 INFO [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f7b113264f1c67c501d9bd1480cc2b7 columnFamilyName B 2024-11-19T00:13:50,119 DEBUG [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:50,120 INFO [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] regionserver.HStore(327): Store=0f7b113264f1c67c501d9bd1480cc2b7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:13:50,120 INFO [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:50,121 INFO [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:13:50,121 INFO [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f7b113264f1c67c501d9bd1480cc2b7 columnFamilyName C 2024-11-19T00:13:50,121 DEBUG [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:50,122 INFO [StoreOpener-0f7b113264f1c67c501d9bd1480cc2b7-1 {}] regionserver.HStore(327): Store=0f7b113264f1c67c501d9bd1480cc2b7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:13:50,122 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:50,122 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:50,123 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:50,124 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:50,126 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:50,126 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:50,126 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T00:13:50,128 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:50,129 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 0f7b113264f1c67c501d9bd1480cc2b7; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59439038, jitterRate=-0.1142893135547638}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T00:13:50,129 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:50,130 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 0f7b113264f1c67c501d9bd1480cc2b7: Running coprocessor pre-open hook at 1731975230106Writing region info on filesystem at 1731975230106Initializing all the Stores at 1731975230107 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975230107Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975230108 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975230108Cleaning up temporary data from old regions at 1731975230126 (+18 ms)Running coprocessor post-open hooks at 1731975230129 (+3 ms)Region opened successfully at 1731975230130 (+1 ms) 2024-11-19T00:13:50,131 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., pid=43, masterSystemTime=1731975230102 2024-11-19T00:13:50,133 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:50,133 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:50,134 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=0f7b113264f1c67c501d9bd1480cc2b7, regionState=OPEN, openSeqNum=5, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:13:50,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 because future has completed 2024-11-19T00:13:50,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=41 2024-11-19T00:13:50,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=41, state=SUCCESS, hasLock=false; OpenRegionProcedure 0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 in 188 msec 2024-11-19T00:13:50,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-19T00:13:50,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f7b113264f1c67c501d9bd1480cc2b7, REOPEN/MOVE in 518 msec 2024-11-19T00:13:50,145 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-19T00:13:50,145 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 530 msec 2024-11-19T00:13:50,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 968 msec 2024-11-19T00:13:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-19T00:13:50,156 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:13:50,158 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,158 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:13:50,158 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:13:50,160 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:13:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:13:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:13:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f949c23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:13:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:13:50,162 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:50,163 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:13:50,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bb77e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,165 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:13:50,165 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:13:50,166 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:13:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:13:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:13:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b8288ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:13:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:13:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:50,169 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:13:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3444ede6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,170 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,171 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:13:50,171 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:13:50,171 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:13:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:13:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:13:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1917ad40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:13:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:13:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:50,173 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:13:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7afae2f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,175 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:13:50,175 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:13:50,176 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:13:50,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:13:50,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:13:50,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e1e749b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:13:50,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:13:50,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:50,179 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:13:50,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38807829, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,181 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:13:50,181 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:13:50,182 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:13:50,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:13:50,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:13:50,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72a43628, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:13:50,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:13:50,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:50,185 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:13:50,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c0d17eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,187 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:13:50,187 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:13:50,188 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:13:50,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:13:50,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:13:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3226ea5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:13:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:13:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:50,190 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:13:50,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d5f36b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,192 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,192 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:13:50,192 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:13:50,193 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:13:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:13:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:13:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fa74f11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:13:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:13:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:50,195 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:13:50,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2966ad20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,197 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,197 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:13:50,197 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:13:50,198 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:13:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:13:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:13:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763676f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:13:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:13:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:50,200 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:13:50,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e473665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,202 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,202 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:13:50,202 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:13:50,203 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:13:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:13:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:13:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53e6e1f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:13:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:13:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:13:50,205 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:13:50,206 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13e246ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:13:50,206 DEBUG [Thread-622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:13:50,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:13:50,208 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:13:50,208 DEBUG [Thread-628 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:13:50,208 DEBUG [Thread-615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:13:50,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:13:50,209 DEBUG [Thread-624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:13:50,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-19T00:13:50,210 DEBUG [Thread-613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:13:50,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:13:50,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:13:50,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:13:50,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:13:50,210 DEBUG [Thread-619 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:13:50,211 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:13:50,211 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:13:50,210 DEBUG [Thread-626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:13:50,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T00:13:50,212 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:13:50,212 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:13:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:13:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:13:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:13:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:13:50,213 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:13:50,211 DEBUG [Thread-611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:13:50,211 DEBUG [Thread-617 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:13:50,211 DEBUG [Thread-613 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:13:50,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:13:50,214 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:13:50,214 DEBUG [Thread-613 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:13:50,214 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:13:50,215 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:13:50,215 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:13:50,215 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:13:50,216 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:13:50,216 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:13:50,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:13:50,216 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:13:50,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:13:50,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:13:50,218 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:13:50,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:13:50,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:13:50,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:13:50,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:13:50,222 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:13:50,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:13:50,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:13:50,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:13:50,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:13:50,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:50,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f7b113264f1c67c501d9bd1480cc2b7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T00:13:50,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=A 2024-11-19T00:13:50,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:50,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=B 2024-11-19T00:13:50,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:50,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=C 2024-11-19T00:13:50,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:50,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T00:13:50,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731975230295, max timestamp=1731975230335Num uniques 859; 2024-11-19T00:13:50,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T00:13:50,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119388d82665d8b4e17aa4640130646148d_0f7b113264f1c67c501d9bd1480cc2b7 is 50, key is test_row_0/A:col10/1731975230267/Put/seqid=0 2024-11-19T00:13:50,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731975230295, max timestamp=1731975230335Num uniques 859; 2024-11-19T00:13:50,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T00:13:50,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975230295, max timestamp=1731975230346Num uniques 859; 2024-11-19T00:13:50,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:13:50,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T00:13:50,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:50,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:13:50,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:50,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:50,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:50,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:50,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741945_1121 (size=14594) 2024-11-19T00:13:50,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:50,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975290398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:50,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:50,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:50,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46992 deadline: 1731975290399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:50,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:50,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:50,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975290410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:50,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:50,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47048 deadline: 1731975290413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:50,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:50,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:50,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47040 deadline: 1731975290416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:50,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:50,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T00:13:50,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:50,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:13:50,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:50,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:50,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:50,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:50,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:50,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975290524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:50,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46992 deadline: 1731975290525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:50,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:50,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T00:13:50,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:50,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975290534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:50,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:50,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:50,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47040 deadline: 1731975290534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:50,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:50,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:50,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47048 deadline: 1731975290536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:50,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:50,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T00:13:50,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:50,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:13:50,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:50,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:50,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:50,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975290744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:50,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:50,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46992 deadline: 1731975290744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:50,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:50,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:50,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975290754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47048 deadline: 1731975290754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:50,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47040 deadline: 1731975290755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:50,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:50,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:50,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:50,791 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:50,800 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119388d82665d8b4e17aa4640130646148d_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119388d82665d8b4e17aa4640130646148d_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:50,806 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/0a1ee3b775c3400585e13dfd60e0ea16, store: [table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:50,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/0a1ee3b775c3400585e13dfd60e0ea16 is 175, key is test_row_0/A:col10/1731975230267/Put/seqid=0 2024-11-19T00:13:50,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741946_1122 (size=39549) 2024-11-19T00:13:50,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T00:13:50,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:50,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:13:50,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:50,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:50,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:50,844 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/0a1ee3b775c3400585e13dfd60e0ea16 2024-11-19T00:13:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T00:13:50,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/cc0531ad622d45f484d16d4b544598b8 is 50, key is test_row_0/B:col10/1731975230267/Put/seqid=0 2024-11-19T00:13:50,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741947_1123 (size=12001) 2024-11-19T00:13:50,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T00:13:50,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:50,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:13:50,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:50,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:50,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:51,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975291054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:51,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:51,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:51,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975291064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:51,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46992 deadline: 1731975291064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:51,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47048 deadline: 1731975291064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:51,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47040 deadline: 1731975291064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:51,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T00:13:51,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:51,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:13:51,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:51,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:51,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:51,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/cc0531ad622d45f484d16d4b544598b8 2024-11-19T00:13:51,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T00:13:51,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:51,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:13:51,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:51,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:51,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:51,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/c0591aa69ea8442ebbd46f924b78ac79 is 50, key is test_row_0/C:col10/1731975230267/Put/seqid=0 2024-11-19T00:13:51,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741948_1124 (size=12001) 2024-11-19T00:13:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T00:13:51,361 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T00:13:51,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T00:13:51,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:51,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:13:51,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:51,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:51,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:51,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975291564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:51,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:51,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:51,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:51,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47048 deadline: 1731975291574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:51,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46992 deadline: 1731975291583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:51,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:51,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47040 deadline: 1731975291583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:51,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:51,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:51,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975291584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:51,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T00:13:51,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:51,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:13:51,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:51,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:51,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:51,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/c0591aa69ea8442ebbd46f924b78ac79 2024-11-19T00:13:51,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/0a1ee3b775c3400585e13dfd60e0ea16 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/0a1ee3b775c3400585e13dfd60e0ea16 2024-11-19T00:13:51,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/0a1ee3b775c3400585e13dfd60e0ea16, entries=200, sequenceid=14, filesize=38.6 K 2024-11-19T00:13:51,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T00:13:51,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:51,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:13:51,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:51,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:51,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/cc0531ad622d45f484d16d4b544598b8 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/cc0531ad622d45f484d16d4b544598b8 2024-11-19T00:13:51,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:51,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/cc0531ad622d45f484d16d4b544598b8, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T00:13:51,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/c0591aa69ea8442ebbd46f924b78ac79 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/c0591aa69ea8442ebbd46f924b78ac79 2024-11-19T00:13:51,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/c0591aa69ea8442ebbd46f924b78ac79, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T00:13:51,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for 0f7b113264f1c67c501d9bd1480cc2b7 in 1523ms, sequenceid=14, compaction requested=false 2024-11-19T00:13:51,793 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T00:13:51,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:13:51,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T00:13:51,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:51,929 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 0f7b113264f1c67c501d9bd1480cc2b7 3/3 column families, dataSize=136.42 KB heapSize=385.38 KB 2024-11-19T00:13:51,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=A 2024-11-19T00:13:51,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:51,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=B 2024-11-19T00:13:51,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:51,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=C 2024-11-19T00:13:51,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:51,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190000385f26f647eaa6fd726118f50ea3_0f7b113264f1c67c501d9bd1480cc2b7 is 50, key is test_row_0/A:col10/1731975230412/Put/seqid=0 2024-11-19T00:13:51,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741949_1125 (size=12154) 2024-11-19T00:13:51,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:51,964 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190000385f26f647eaa6fd726118f50ea3_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190000385f26f647eaa6fd726118f50ea3_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:51,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/feef0c23e2d844aaa475e3cc99f88065, store: [table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:51,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/feef0c23e2d844aaa475e3cc99f88065 is 175, key is test_row_0/A:col10/1731975230412/Put/seqid=0 2024-11-19T00:13:51,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741950_1126 (size=30955) 2024-11-19T00:13:51,983 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/feef0c23e2d844aaa475e3cc99f88065 2024-11-19T00:13:51,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/b55d3cbb0d1e4574b9551c42ad1cf006 is 50, key is test_row_0/B:col10/1731975230412/Put/seqid=0 2024-11-19T00:13:52,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741951_1127 (size=12001) 2024-11-19T00:13:52,006 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/b55d3cbb0d1e4574b9551c42ad1cf006 2024-11-19T00:13:52,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/b0aad806c5ec47afaf987dde33af38e4 is 50, key is test_row_0/C:col10/1731975230412/Put/seqid=0 2024-11-19T00:13:52,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741952_1128 (size=12001) 2024-11-19T00:13:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T00:13:52,429 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/b0aad806c5ec47afaf987dde33af38e4 2024-11-19T00:13:52,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/feef0c23e2d844aaa475e3cc99f88065 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/feef0c23e2d844aaa475e3cc99f88065 2024-11-19T00:13:52,441 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/feef0c23e2d844aaa475e3cc99f88065, entries=150, sequenceid=50, filesize=30.2 K 2024-11-19T00:13:52,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/b55d3cbb0d1e4574b9551c42ad1cf006 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/b55d3cbb0d1e4574b9551c42ad1cf006 2024-11-19T00:13:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,449 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/b55d3cbb0d1e4574b9551c42ad1cf006, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T00:13:52,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/b0aad806c5ec47afaf987dde33af38e4 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/b0aad806c5ec47afaf987dde33af38e4 2024-11-19T00:13:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,464 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/b0aad806c5ec47afaf987dde33af38e4, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T00:13:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,465 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~385.33 KB/394576, currentSize=0 B/0 for 0f7b113264f1c67c501d9bd1480cc2b7 in 537ms, sequenceid=50, compaction requested=false 2024-11-19T00:13:52,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:13:52,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:52,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-19T00:13:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-19T00:13:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-19T00:13:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2550 sec 2024-11-19T00:13:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.2620 sec 2024-11-19T00:13:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:52,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f7b113264f1c67c501d9bd1480cc2b7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T00:13:52,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=A 2024-11-19T00:13:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:52,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=B 2024-11-19T00:13:52,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:52,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=C 2024-11-19T00:13:52,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193e050108a25c42cfbdb6f561c1067d34_0f7b113264f1c67c501d9bd1480cc2b7 is 50, key is test_row_0/A:col10/1731975232642/Put/seqid=0 2024-11-19T00:13:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741953_1129 (size=21918) 2024-11-19T00:13:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,675 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,682 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193e050108a25c42cfbdb6f561c1067d34_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193e050108a25c42cfbdb6f561c1067d34_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,683 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/92884ac7249d4cfe965968df850fba50, store: [table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/92884ac7249d4cfe965968df850fba50 is 175, key is test_row_0/A:col10/1731975232642/Put/seqid=0 2024-11-19T00:13:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:52,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731975232671, max timestamp=1731975232699Num uniques 859; 2024-11-19T00:13:52,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T00:13:52,703 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731975232671, max timestamp=1731975232699Num uniques 859; 2024-11-19T00:13:52,703 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T00:13:52,703 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731975232671, max timestamp=1731975232699Num uniques 859; 2024-11-19T00:13:52,703 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T00:13:52,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741954_1130 (size=65323) 2024-11-19T00:13:52,717 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/92884ac7249d4cfe965968df850fba50 2024-11-19T00:13:52,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/ffe1e0a694f34d31b92955d5bf28366b is 50, key is test_row_0/B:col10/1731975232642/Put/seqid=0 2024-11-19T00:13:52,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47040 deadline: 1731975292731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:52,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:52,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:52,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:52,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47048 deadline: 1731975292733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:52,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:52,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:52,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:52,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975292738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:52,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:52,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:52,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:52,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:52,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975292740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46992 deadline: 1731975292741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:52,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741955_1131 (size=12001) 2024-11-19T00:13:52,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/ffe1e0a694f34d31b92955d5bf28366b 2024-11-19T00:13:52,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/a8c12167d2a547ecac9b2e38c8f3127e is 50, key is test_row_0/C:col10/1731975232642/Put/seqid=0 2024-11-19T00:13:52,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741956_1132 (size=12001) 2024-11-19T00:13:52,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/a8c12167d2a547ecac9b2e38c8f3127e 2024-11-19T00:13:52,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/92884ac7249d4cfe965968df850fba50 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/92884ac7249d4cfe965968df850fba50 2024-11-19T00:13:52,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/92884ac7249d4cfe965968df850fba50, entries=350, sequenceid=60, filesize=63.8 K 2024-11-19T00:13:52,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/ffe1e0a694f34d31b92955d5bf28366b as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ffe1e0a694f34d31b92955d5bf28366b 2024-11-19T00:13:52,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ffe1e0a694f34d31b92955d5bf28366b, entries=150, sequenceid=60, filesize=11.7 K 2024-11-19T00:13:52,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/a8c12167d2a547ecac9b2e38c8f3127e as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/a8c12167d2a547ecac9b2e38c8f3127e 2024-11-19T00:13:52,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/a8c12167d2a547ecac9b2e38c8f3127e, entries=150, sequenceid=60, filesize=11.7 K 2024-11-19T00:13:52,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 0f7b113264f1c67c501d9bd1480cc2b7 in 182ms, sequenceid=60, compaction requested=true 2024-11-19T00:13:52,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:13:52,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:13:52,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:52,829 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:52,829 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:52,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:13:52,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:52,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:13:52,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:52,833 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:52,833 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/B is initiating minor compaction (all files) 2024-11-19T00:13:52,833 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/B in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:52,834 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:52,834 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/cc0531ad622d45f484d16d4b544598b8, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/b55d3cbb0d1e4574b9551c42ad1cf006, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ffe1e0a694f34d31b92955d5bf28366b] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=35.2 K 2024-11-19T00:13:52,834 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/A is initiating minor compaction (all files) 2024-11-19T00:13:52,834 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/A in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:52,834 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/0a1ee3b775c3400585e13dfd60e0ea16, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/feef0c23e2d844aaa475e3cc99f88065, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/92884ac7249d4cfe965968df850fba50] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=132.6 K 2024-11-19T00:13:52,834 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:52,834 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. files: [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/0a1ee3b775c3400585e13dfd60e0ea16, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/feef0c23e2d844aaa475e3cc99f88065, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/92884ac7249d4cfe965968df850fba50] 2024-11-19T00:13:52,835 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting cc0531ad622d45f484d16d4b544598b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731975230236 2024-11-19T00:13:52,835 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a1ee3b775c3400585e13dfd60e0ea16, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731975230236 2024-11-19T00:13:52,835 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting b55d3cbb0d1e4574b9551c42ad1cf006, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731975230407 2024-11-19T00:13:52,835 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting feef0c23e2d844aaa475e3cc99f88065, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731975230407 2024-11-19T00:13:52,836 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92884ac7249d4cfe965968df850fba50, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731975232599 2024-11-19T00:13:52,837 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting ffe1e0a694f34d31b92955d5bf28366b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731975232642 2024-11-19T00:13:52,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:52,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f7b113264f1c67c501d9bd1480cc2b7 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-11-19T00:13:52,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731975232728, max timestamp=1731975232849Num uniques 1168; 2024-11-19T00:13:52,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731975232728, max timestamp=1731975232849Num uniques 1168; 2024-11-19T00:13:52,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-11-19T00:13:52,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-11-19T00:13:52,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=A 2024-11-19T00:13:52,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:52,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=B 2024-11-19T00:13:52,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:52,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=C 2024-11-19T00:13:52,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:13:52,864 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#B#compaction#114 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:52,864 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:52,864 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/8d48d6551aad4e56a5e50683c985d6ad is 50, key is test_row_0/B:col10/1731975232642/Put/seqid=0 2024-11-19T00:13:52,871 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119bc907cf139d147169cf31649fd204109_0f7b113264f1c67c501d9bd1480cc2b7 store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:52,874 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119bc907cf139d147169cf31649fd204109_0f7b113264f1c67c501d9bd1480cc2b7, store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:52,874 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bc907cf139d147169cf31649fd204109_0f7b113264f1c67c501d9bd1480cc2b7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:52,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741957_1133 (size=12104) 2024-11-19T00:13:52,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47040 deadline: 1731975292883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:52,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:52,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975292885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:52,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:52,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:52,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:52,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:52,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:52,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741958_1134 (size=4469) 2024-11-19T00:13:52,901 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/8d48d6551aad4e56a5e50683c985d6ad as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/8d48d6551aad4e56a5e50683c985d6ad 2024-11-19T00:13:52,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46992 deadline: 1731975292887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:52,903 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#A#compaction#115 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:52,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975292888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:52,903 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/c4227c9be7cd4112bcea48394b357b0f is 175, key is test_row_0/A:col10/1731975232642/Put/seqid=0 2024-11-19T00:13:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:52,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:52,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47048 deadline: 1731975292894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:52,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:52,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:52,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:52,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:52,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:52,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:52,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111915761b68da49433f927add99462eae6b_0f7b113264f1c67c501d9bd1480cc2b7 is 50, key is test_row_0/A:col10/1731975232739/Put/seqid=0 2024-11-19T00:13:52,929 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/B of 0f7b113264f1c67c501d9bd1480cc2b7 into 8d48d6551aad4e56a5e50683c985d6ad(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:52,929 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:13:52,929 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/B, priority=13, startTime=1731975232829; duration=0sec 2024-11-19T00:13:52,929 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:52,929 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:B 2024-11-19T00:13:52,929 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:52,931 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:52,931 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/C is initiating minor compaction (all files) 2024-11-19T00:13:52,931 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/C in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:52,932 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/c0591aa69ea8442ebbd46f924b78ac79, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/b0aad806c5ec47afaf987dde33af38e4, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/a8c12167d2a547ecac9b2e38c8f3127e] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=35.2 K 2024-11-19T00:13:52,932 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting c0591aa69ea8442ebbd46f924b78ac79, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731975230236 2024-11-19T00:13:52,933 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting b0aad806c5ec47afaf987dde33af38e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731975230407 2024-11-19T00:13:52,933 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting a8c12167d2a547ecac9b2e38c8f3127e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731975232642 2024-11-19T00:13:52,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741959_1135 (size=31058) 2024-11-19T00:13:52,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741960_1136 (size=12154) 2024-11-19T00:13:52,961 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/c4227c9be7cd4112bcea48394b357b0f as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/c4227c9be7cd4112bcea48394b357b0f 2024-11-19T00:13:52,969 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/A of 0f7b113264f1c67c501d9bd1480cc2b7 into c4227c9be7cd4112bcea48394b357b0f(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:52,969 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#C#compaction#117 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:52,970 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:13:52,970 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/A, priority=13, startTime=1731975232829; duration=0sec 2024-11-19T00:13:52,970 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:52,970 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:A 2024-11-19T00:13:52,970 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/e5a79dcb857147c896ca983917e745de is 50, key is test_row_0/C:col10/1731975232642/Put/seqid=0 2024-11-19T00:13:52,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741961_1137 (size=12104) 2024-11-19T00:13:52,992 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/e5a79dcb857147c896ca983917e745de as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/e5a79dcb857147c896ca983917e745de 2024-11-19T00:13:53,002 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/C of 0f7b113264f1c67c501d9bd1480cc2b7 into e5a79dcb857147c896ca983917e745de(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:53,002 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:13:53,002 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/C, priority=13, startTime=1731975232829; duration=0sec 2024-11-19T00:13:53,002 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:53,002 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:C 2024-11-19T00:13:53,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47040 deadline: 1731975293004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975293004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975293014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47048 deadline: 1731975293014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46992 deadline: 1731975293014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47040 deadline: 1731975293214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975293214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975293223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47048 deadline: 1731975293224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46992 deadline: 1731975293224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,362 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:53,366 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111915761b68da49433f927add99462eae6b_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111915761b68da49433f927add99462eae6b_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:53,367 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/ee0f30fb72734e2191c5b1919a8e6b2b, store: [table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:53,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/ee0f30fb72734e2191c5b1919a8e6b2b is 175, key is test_row_0/A:col10/1731975232739/Put/seqid=0 2024-11-19T00:13:53,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741962_1138 (size=30955) 2024-11-19T00:13:53,378 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/ee0f30fb72734e2191c5b1919a8e6b2b 2024-11-19T00:13:53,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/873d75a651b842039154d8b3669bed27 is 50, key is test_row_0/B:col10/1731975232739/Put/seqid=0 2024-11-19T00:13:53,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741963_1139 (size=12001) 2024-11-19T00:13:53,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/873d75a651b842039154d8b3669bed27 2024-11-19T00:13:53,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/0e63459c39e44440a479838c99f62b8f is 50, key is test_row_0/C:col10/1731975232739/Put/seqid=0 2024-11-19T00:13:53,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741964_1140 (size=12001) 2024-11-19T00:13:53,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/0e63459c39e44440a479838c99f62b8f 2024-11-19T00:13:53,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/ee0f30fb72734e2191c5b1919a8e6b2b as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/ee0f30fb72734e2191c5b1919a8e6b2b 2024-11-19T00:13:53,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/ee0f30fb72734e2191c5b1919a8e6b2b, entries=150, sequenceid=98, filesize=30.2 K 2024-11-19T00:13:53,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/873d75a651b842039154d8b3669bed27 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/873d75a651b842039154d8b3669bed27 2024-11-19T00:13:53,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/873d75a651b842039154d8b3669bed27, entries=150, sequenceid=98, filesize=11.7 K 2024-11-19T00:13:53,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/0e63459c39e44440a479838c99f62b8f as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/0e63459c39e44440a479838c99f62b8f 2024-11-19T00:13:53,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/0e63459c39e44440a479838c99f62b8f, entries=150, sequenceid=98, filesize=11.7 K 2024-11-19T00:13:53,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~76.04 KB/77860, heapSize ~204.14 KB/209040, currentSize=107.34 KB/109920 for 0f7b113264f1c67c501d9bd1480cc2b7 in 632ms, sequenceid=98, compaction requested=false 2024-11-19T00:13:53,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:13:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:53,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f7b113264f1c67c501d9bd1480cc2b7 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T00:13:53,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=A 2024-11-19T00:13:53,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:53,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=B 2024-11-19T00:13:53,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:53,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=C 2024-11-19T00:13:53,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:53,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975293539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47040 deadline: 1731975293540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46992 deadline: 1731975293541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47048 deadline: 1731975293544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975293550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192f64c7773d54474fa29c13f844dfb592_0f7b113264f1c67c501d9bd1480cc2b7 is 50, key is test_row_0/A:col10/1731975232882/Put/seqid=0 2024-11-19T00:13:53,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741965_1141 (size=14594) 2024-11-19T00:13:53,568 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:53,574 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192f64c7773d54474fa29c13f844dfb592_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192f64c7773d54474fa29c13f844dfb592_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:53,577 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/632ca0d1991741288e4af821cdef87f7, store: [table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:53,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/632ca0d1991741288e4af821cdef87f7 is 175, key is test_row_0/A:col10/1731975232882/Put/seqid=0 2024-11-19T00:13:53,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741966_1142 (size=39549) 2024-11-19T00:13:53,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975293654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46992 deadline: 1731975293654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47040 deadline: 1731975293654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47048 deadline: 1731975293654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975293664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975293863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46992 deadline: 1731975293864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47040 deadline: 1731975293864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47048 deadline: 1731975293864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:53,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975293874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:53,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:53,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:53,985 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/632ca0d1991741288e4af821cdef87f7 2024-11-19T00:13:53,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/7c8f70aaee1d45718c31cad1ac2a9288 is 50, key is test_row_0/B:col10/1731975232882/Put/seqid=0 2024-11-19T00:13:54,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741967_1143 (size=12001) 2024-11-19T00:13:54,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975294174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:54,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46992 deadline: 1731975294174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:54,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47040 deadline: 1731975294174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47048 deadline: 1731975294174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:54,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975294183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:54,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T00:13:54,384 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:13:54,385 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:13:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-19T00:13:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T00:13:54,387 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:13:54,388 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:13:54,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:13:54,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/7c8f70aaee1d45718c31cad1ac2a9288 2024-11-19T00:13:54,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/4cc302c5e5f74eca95368852721a5c3f is 50, key is test_row_0/C:col10/1731975232882/Put/seqid=0 2024-11-19T00:13:54,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741968_1144 (size=12001) 2024-11-19T00:13:54,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/4cc302c5e5f74eca95368852721a5c3f 2024-11-19T00:13:54,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/632ca0d1991741288e4af821cdef87f7 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/632ca0d1991741288e4af821cdef87f7 2024-11-19T00:13:54,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/632ca0d1991741288e4af821cdef87f7, entries=200, sequenceid=121, filesize=38.6 K 2024-11-19T00:13:54,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/7c8f70aaee1d45718c31cad1ac2a9288 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/7c8f70aaee1d45718c31cad1ac2a9288 2024-11-19T00:13:54,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/7c8f70aaee1d45718c31cad1ac2a9288, entries=150, sequenceid=121, filesize=11.7 K 2024-11-19T00:13:54,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/4cc302c5e5f74eca95368852721a5c3f as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/4cc302c5e5f74eca95368852721a5c3f 2024-11-19T00:13:54,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/4cc302c5e5f74eca95368852721a5c3f, entries=150, sequenceid=121, filesize=11.7 K 2024-11-19T00:13:54,454 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 0f7b113264f1c67c501d9bd1480cc2b7 in 928ms, sequenceid=121, compaction requested=true 2024-11-19T00:13:54,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:13:54,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:13:54,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:54,454 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:54,454 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:54,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:13:54,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:54,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:13:54,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:54,457 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:54,457 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:54,457 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/B is initiating minor compaction (all files) 2024-11-19T00:13:54,457 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/A is initiating minor compaction (all files) 2024-11-19T00:13:54,457 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/A in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:54,457 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/B in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:54,457 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/8d48d6551aad4e56a5e50683c985d6ad, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/873d75a651b842039154d8b3669bed27, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/7c8f70aaee1d45718c31cad1ac2a9288] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=35.3 K 2024-11-19T00:13:54,457 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/c4227c9be7cd4112bcea48394b357b0f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/ee0f30fb72734e2191c5b1919a8e6b2b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/632ca0d1991741288e4af821cdef87f7] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=99.2 K 2024-11-19T00:13:54,457 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:54,457 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. files: [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/c4227c9be7cd4112bcea48394b357b0f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/ee0f30fb72734e2191c5b1919a8e6b2b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/632ca0d1991741288e4af821cdef87f7] 2024-11-19T00:13:54,458 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d48d6551aad4e56a5e50683c985d6ad, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731975232642 2024-11-19T00:13:54,458 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4227c9be7cd4112bcea48394b357b0f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731975232642 2024-11-19T00:13:54,466 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee0f30fb72734e2191c5b1919a8e6b2b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731975232739 2024-11-19T00:13:54,466 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 873d75a651b842039154d8b3669bed27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731975232739 2024-11-19T00:13:54,467 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c8f70aaee1d45718c31cad1ac2a9288, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731975232880 2024-11-19T00:13:54,467 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 632ca0d1991741288e4af821cdef87f7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731975232880 2024-11-19T00:13:54,484 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#B#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:54,485 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/0cc530bd1d684baca741d0859faf821b is 50, key is test_row_0/B:col10/1731975232882/Put/seqid=0 2024-11-19T00:13:54,487 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T00:13:54,504 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111957f40e0cdf88430196bcd8e1b8fed9a0_0f7b113264f1c67c501d9bd1480cc2b7 store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:54,507 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111957f40e0cdf88430196bcd8e1b8fed9a0_0f7b113264f1c67c501d9bd1480cc2b7, store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:54,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741969_1145 (size=12207) 2024-11-19T00:13:54,508 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111957f40e0cdf88430196bcd8e1b8fed9a0_0f7b113264f1c67c501d9bd1480cc2b7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:54,517 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/0cc530bd1d684baca741d0859faf821b as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/0cc530bd1d684baca741d0859faf821b 2024-11-19T00:13:54,526 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/B of 0f7b113264f1c67c501d9bd1480cc2b7 into 0cc530bd1d684baca741d0859faf821b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:54,526 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:13:54,526 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/B, priority=13, startTime=1731975234454; duration=0sec 2024-11-19T00:13:54,526 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:54,526 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:B 2024-11-19T00:13:54,526 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:54,528 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:54,528 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/C is initiating minor compaction (all files) 2024-11-19T00:13:54,528 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/C in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:54,529 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/e5a79dcb857147c896ca983917e745de, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/0e63459c39e44440a479838c99f62b8f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/4cc302c5e5f74eca95368852721a5c3f] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=35.3 K 2024-11-19T00:13:54,531 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting e5a79dcb857147c896ca983917e745de, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731975232642 2024-11-19T00:13:54,532 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e63459c39e44440a479838c99f62b8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731975232739 2024-11-19T00:13:54,536 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cc302c5e5f74eca95368852721a5c3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731975232880 2024-11-19T00:13:54,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741970_1146 (size=4469) 2024-11-19T00:13:54,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T00:13:54,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:54,541 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 0f7b113264f1c67c501d9bd1480cc2b7 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T00:13:54,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=A 2024-11-19T00:13:54,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:54,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=B 2024-11-19T00:13:54,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:54,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=C 2024-11-19T00:13:54,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:54,542 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#A#compaction#124 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:54,543 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/52dca9e672c345d595731b81195fa486 is 175, key is test_row_0/A:col10/1731975232882/Put/seqid=0 2024-11-19T00:13:54,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741971_1147 (size=31161) 2024-11-19T00:13:54,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192fb0938e06784cf9a023d1521e339580_0f7b113264f1c67c501d9bd1480cc2b7 is 50, key is test_row_0/A:col10/1731975233530/Put/seqid=0 2024-11-19T00:13:54,561 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#C#compaction#126 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:54,561 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/31023d0047214ef19aa9f1ebd47a5204 is 50, key is test_row_0/C:col10/1731975232882/Put/seqid=0 2024-11-19T00:13:54,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741973_1149 (size=12207) 2024-11-19T00:13:54,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741972_1148 (size=12254) 2024-11-19T00:13:54,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:54,584 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192fb0938e06784cf9a023d1521e339580_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192fb0938e06784cf9a023d1521e339580_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:54,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/0a30671bd74e4604bb64c449dfc6d0ab, store: [table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:54,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/0a30671bd74e4604bb64c449dfc6d0ab is 175, key is test_row_0/A:col10/1731975233530/Put/seqid=0 2024-11-19T00:13:54,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741974_1150 (size=31055) 2024-11-19T00:13:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:54,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:13:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T00:13:54,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47040 deadline: 1731975294709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:54,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16434, totalHeapSize=37576, min timestamp=1731975234694, max timestamp=1731975234715Num uniques 859; 2024-11-19T00:13:54,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=16434, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T00:13:54,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16434, totalHeapSize=37576, min timestamp=1731975234694, max timestamp=1731975234715Num uniques 859; 2024-11-19T00:13:54,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=16434, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T00:13:54,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16434, totalHeapSize=37576, min timestamp=1731975234694, max timestamp=1731975234715Num uniques 859; 2024-11-19T00:13:54,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=16434, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T00:13:54,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975294746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:54,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975294747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:54,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47048 deadline: 1731975294749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:54,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46992 deadline: 1731975294749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:54,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:54,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:54,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:54,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:54,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47040 deadline: 1731975294824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:54,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975294864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:54,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47048 deadline: 1731975294864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:54,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975294864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:54,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46992 deadline: 1731975294864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:54,993 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/0a30671bd74e4604bb64c449dfc6d0ab 2024-11-19T00:13:55,000 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/52dca9e672c345d595731b81195fa486 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/52dca9e672c345d595731b81195fa486 2024-11-19T00:13:55,013 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/31023d0047214ef19aa9f1ebd47a5204 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/31023d0047214ef19aa9f1ebd47a5204 2024-11-19T00:13:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T00:13:55,015 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/A of 0f7b113264f1c67c501d9bd1480cc2b7 into 52dca9e672c345d595731b81195fa486(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:55,015 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:13:55,015 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/A, priority=13, startTime=1731975234454; duration=0sec 2024-11-19T00:13:55,015 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:55,015 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:A 2024-11-19T00:13:55,024 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/C of 0f7b113264f1c67c501d9bd1480cc2b7 into 31023d0047214ef19aa9f1ebd47a5204(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:55,024 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:13:55,024 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/C, priority=13, startTime=1731975234455; duration=0sec 2024-11-19T00:13:55,025 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:55,025 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:C 2024-11-19T00:13:55,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/ddf85a6e26274765b51afdbf57c44172 is 50, key is test_row_0/B:col10/1731975233530/Put/seqid=0 2024-11-19T00:13:55,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47040 deadline: 1731975295034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:55,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:55,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741975_1151 (size=12101) 2024-11-19T00:13:55,042 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/ddf85a6e26274765b51afdbf57c44172 2024-11-19T00:13:55,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/42c34a9398e7468c9833ce0c4efdb09e is 50, key is test_row_0/C:col10/1731975233530/Put/seqid=0 2024-11-19T00:13:55,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741976_1152 (size=12101) 2024-11-19T00:13:55,054 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/42c34a9398e7468c9833ce0c4efdb09e 2024-11-19T00:13:55,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/0a30671bd74e4604bb64c449dfc6d0ab as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/0a30671bd74e4604bb64c449dfc6d0ab 2024-11-19T00:13:55,064 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/0a30671bd74e4604bb64c449dfc6d0ab, entries=150, sequenceid=134, filesize=30.3 K 2024-11-19T00:13:55,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/ddf85a6e26274765b51afdbf57c44172 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ddf85a6e26274765b51afdbf57c44172 2024-11-19T00:13:55,070 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ddf85a6e26274765b51afdbf57c44172, entries=150, sequenceid=134, filesize=11.8 K 2024-11-19T00:13:55,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/42c34a9398e7468c9833ce0c4efdb09e as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/42c34a9398e7468c9833ce0c4efdb09e 2024-11-19T00:13:55,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975295074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:55,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47048 deadline: 1731975295074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46992 deadline: 1731975295074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:55,076 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/42c34a9398e7468c9833ce0c4efdb09e, entries=150, sequenceid=134, filesize=11.8 K 2024-11-19T00:13:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:55,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975295075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:55,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,077 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=134.18 KB/137400 for 0f7b113264f1c67c501d9bd1480cc2b7 in 536ms, sequenceid=134, compaction requested=false 2024-11-19T00:13:55,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:55,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:13:55,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:55,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-19T00:13:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-19T00:13:55,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-19T00:13:55,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 691 msec 2024-11-19T00:13:55,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 695 msec 2024-11-19T00:13:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:55,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f7b113264f1c67c501d9bd1480cc2b7 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-19T00:13:55,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=A 2024-11-19T00:13:55,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:55,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=B 2024-11-19T00:13:55,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:55,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=C 2024-11-19T00:13:55,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:55,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fd3adb4176dc4a2b89514ee2b2336955_0f7b113264f1c67c501d9bd1480cc2b7 is 50, key is test_row_0/A:col10/1731975235346/Put/seqid=0 2024-11-19T00:13:55,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:55,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:55,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975295384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:55,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975295384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:55,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:55,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47040 deadline: 1731975295385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:55,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:55,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741977_1153 (size=14794) 2024-11-19T00:13:55,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46992 deadline: 1731975295391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:55,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:55,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47048 deadline: 1731975295392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:55,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:55,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47040 deadline: 1731975295504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:55,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:55,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47048 deadline: 1731975295504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:55,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:55,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46992 deadline: 1731975295504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:55,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:55,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:55,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T00:13:55,524 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:13:55,524 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:13:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-19T00:13:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T00:13:55,542 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:13:55,544 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:13:55,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:13:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T00:13:55,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T00:13:55,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:55,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:13:55,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:55,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:55,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:55,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47040 deadline: 1731975295714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:55,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47048 deadline: 1731975295714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:55,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:55,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:55,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46992 deadline: 1731975295723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:55,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:55,796 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:55,802 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fd3adb4176dc4a2b89514ee2b2336955_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fd3adb4176dc4a2b89514ee2b2336955_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:55,804 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/2d991441ae51493bbec14b3a7815fd6c, store: [table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:55,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/2d991441ae51493bbec14b3a7815fd6c is 175, key is test_row_0/A:col10/1731975235346/Put/seqid=0 2024-11-19T00:13:55,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741978_1154 (size=39749) 2024-11-19T00:13:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T00:13:55,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T00:13:55,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:55,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:13:55,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:55,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:55,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:55,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:55,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975295904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975295904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:55,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:55,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:55,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:56,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T00:13:56,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:56,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:13:56,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:56,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:56,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:56,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47048 deadline: 1731975296024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:56,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:56,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:56,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47040 deadline: 1731975296024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:56,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:56,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:56,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:56,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46992 deadline: 1731975296034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T00:13:56,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T00:13:56,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:56,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:13:56,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:56,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:56,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:56,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:56,241 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/2d991441ae51493bbec14b3a7815fd6c 2024-11-19T00:13:56,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/99ee6915b68c4b6689cc33ccdc2828f6 is 50, key is test_row_0/B:col10/1731975235346/Put/seqid=0 2024-11-19T00:13:56,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741979_1155 (size=12151) 2024-11-19T00:13:56,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T00:13:56,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:56,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:13:56,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:56,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:56,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:56,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:56,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T00:13:56,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:56,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:13:56,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:56,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:56,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:56,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:56,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47048 deadline: 1731975296538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:56,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:56,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:56,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:56,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46992 deadline: 1731975296544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:56,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47040 deadline: 1731975296544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:56,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T00:13:56,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:56,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:13:56,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:56,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:56,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:56,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/99ee6915b68c4b6689cc33ccdc2828f6 2024-11-19T00:13:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T00:13:56,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/6e6d74878553400bb9220a2b465fb404 is 50, key is test_row_0/C:col10/1731975235346/Put/seqid=0 2024-11-19T00:13:56,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741980_1156 (size=12151) 2024-11-19T00:13:56,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/6e6d74878553400bb9220a2b465fb404 2024-11-19T00:13:56,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/2d991441ae51493bbec14b3a7815fd6c as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/2d991441ae51493bbec14b3a7815fd6c 2024-11-19T00:13:56,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/2d991441ae51493bbec14b3a7815fd6c, entries=200, sequenceid=170, filesize=38.8 K 2024-11-19T00:13:56,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/99ee6915b68c4b6689cc33ccdc2828f6 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/99ee6915b68c4b6689cc33ccdc2828f6 2024-11-19T00:13:56,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/99ee6915b68c4b6689cc33ccdc2828f6, entries=150, sequenceid=170, filesize=11.9 K 2024-11-19T00:13:56,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/6e6d74878553400bb9220a2b465fb404 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/6e6d74878553400bb9220a2b465fb404 2024-11-19T00:13:56,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/6e6d74878553400bb9220a2b465fb404, entries=150, sequenceid=170, filesize=11.9 K 2024-11-19T00:13:56,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for 0f7b113264f1c67c501d9bd1480cc2b7 in 1420ms, sequenceid=170, compaction requested=true 2024-11-19T00:13:56,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:13:56,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:13:56,768 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:56,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:56,768 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:56,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:13:56,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:56,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:13:56,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:56,770 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:56,770 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/A is initiating minor compaction (all files) 2024-11-19T00:13:56,770 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/A in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:56,770 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/52dca9e672c345d595731b81195fa486, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/0a30671bd74e4604bb64c449dfc6d0ab, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/2d991441ae51493bbec14b3a7815fd6c] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=99.6 K 2024-11-19T00:13:56,770 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:56,770 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. files: [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/52dca9e672c345d595731b81195fa486, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/0a30671bd74e4604bb64c449dfc6d0ab, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/2d991441ae51493bbec14b3a7815fd6c] 2024-11-19T00:13:56,771 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:56,771 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/B is initiating minor compaction (all files) 2024-11-19T00:13:56,771 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/B in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:56,771 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/0cc530bd1d684baca741d0859faf821b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ddf85a6e26274765b51afdbf57c44172, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/99ee6915b68c4b6689cc33ccdc2828f6] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=35.6 K 2024-11-19T00:13:56,771 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52dca9e672c345d595731b81195fa486, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731975232880 2024-11-19T00:13:56,772 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a30671bd74e4604bb64c449dfc6d0ab, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731975233530 2024-11-19T00:13:56,772 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cc530bd1d684baca741d0859faf821b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731975232880 2024-11-19T00:13:56,772 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d991441ae51493bbec14b3a7815fd6c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731975234742 2024-11-19T00:13:56,772 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting ddf85a6e26274765b51afdbf57c44172, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731975233530 2024-11-19T00:13:56,775 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 99ee6915b68c4b6689cc33ccdc2828f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731975234746 2024-11-19T00:13:56,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T00:13:56,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:56,785 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 0f7b113264f1c67c501d9bd1480cc2b7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T00:13:56,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=A 2024-11-19T00:13:56,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:56,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=B 2024-11-19T00:13:56,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:56,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=C 2024-11-19T00:13:56,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:56,797 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#B#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:56,798 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/79d8469c250141a6ae7be93af1115278 is 50, key is test_row_0/B:col10/1731975235346/Put/seqid=0 2024-11-19T00:13:56,815 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:56,832 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119aecc397b64184c7c9143a5dbf9f1998c_0f7b113264f1c67c501d9bd1480cc2b7 store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:56,833 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119aecc397b64184c7c9143a5dbf9f1998c_0f7b113264f1c67c501d9bd1480cc2b7, store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:56,834 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119aecc397b64184c7c9143a5dbf9f1998c_0f7b113264f1c67c501d9bd1480cc2b7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:56,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741981_1157 (size=12459) 2024-11-19T00:13:56,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b741d92d752b461c986dcb36680f27a0_0f7b113264f1c67c501d9bd1480cc2b7 is 50, key is test_row_0/A:col10/1731975235382/Put/seqid=0 2024-11-19T00:13:56,869 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/79d8469c250141a6ae7be93af1115278 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/79d8469c250141a6ae7be93af1115278 2024-11-19T00:13:56,877 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/B of 0f7b113264f1c67c501d9bd1480cc2b7 into 79d8469c250141a6ae7be93af1115278(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:56,877 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:13:56,877 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/B, priority=13, startTime=1731975236768; duration=0sec 2024-11-19T00:13:56,877 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:56,877 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:B 2024-11-19T00:13:56,877 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:56,879 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:56,879 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/C is initiating minor compaction (all files) 2024-11-19T00:13:56,879 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/C in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:56,879 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/31023d0047214ef19aa9f1ebd47a5204, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/42c34a9398e7468c9833ce0c4efdb09e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/6e6d74878553400bb9220a2b465fb404] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=35.6 K 2024-11-19T00:13:56,880 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 31023d0047214ef19aa9f1ebd47a5204, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731975232880 2024-11-19T00:13:56,880 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 42c34a9398e7468c9833ce0c4efdb09e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731975233530 2024-11-19T00:13:56,881 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e6d74878553400bb9220a2b465fb404, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731975234746 2024-11-19T00:13:56,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741982_1158 (size=4469) 2024-11-19T00:13:56,894 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#A#compaction#133 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:56,895 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/b7c7b39909894d638ec3bd697f6ecb32 is 175, key is test_row_0/A:col10/1731975235346/Put/seqid=0 2024-11-19T00:13:56,898 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#C#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:56,900 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/e219cb785e914fb6bdd12a486db80abb is 50, key is test_row_0/C:col10/1731975235346/Put/seqid=0 2024-11-19T00:13:56,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741983_1159 (size=12304) 2024-11-19T00:13:56,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:56,920 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b741d92d752b461c986dcb36680f27a0_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b741d92d752b461c986dcb36680f27a0_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:56,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/779a9c63f4dc42afa0bf4766b5d50bda, store: [table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:56,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/779a9c63f4dc42afa0bf4766b5d50bda is 175, key is test_row_0/A:col10/1731975235382/Put/seqid=0 2024-11-19T00:13:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:56,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:13:56,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741985_1161 (size=12459) 2024-11-19T00:13:56,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741984_1160 (size=31413) 2024-11-19T00:13:56,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741986_1162 (size=31105) 2024-11-19T00:13:57,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731975236965, max timestamp=1731975237002Num uniques 859; 2024-11-19T00:13:57,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T00:13:57,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731975236965, max timestamp=1731975237002Num uniques 859; 2024-11-19T00:13:57,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731975236965, max timestamp=1731975237002Num uniques 859; 2024-11-19T00:13:57,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T00:13:57,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T00:13:57,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975297108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:57,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:57,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:57,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:57,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975297109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:57,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975297224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:57,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975297224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:57,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:57,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:57,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:57,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:57,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:57,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:57,368 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/e219cb785e914fb6bdd12a486db80abb as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/e219cb785e914fb6bdd12a486db80abb 2024-11-19T00:13:57,376 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/C of 0f7b113264f1c67c501d9bd1480cc2b7 into e219cb785e914fb6bdd12a486db80abb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:57,376 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:13:57,376 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/C, priority=13, startTime=1731975236769; duration=0sec 2024-11-19T00:13:57,377 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:57,377 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:C 2024-11-19T00:13:57,386 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/779a9c63f4dc42afa0bf4766b5d50bda 2024-11-19T00:13:57,389 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/b7c7b39909894d638ec3bd697f6ecb32 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/b7c7b39909894d638ec3bd697f6ecb32 2024-11-19T00:13:57,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/319dd584a3d64c6abe9d09f70d78543c is 50, key is test_row_0/B:col10/1731975235382/Put/seqid=0 2024-11-19T00:13:57,403 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/A of 0f7b113264f1c67c501d9bd1480cc2b7 into b7c7b39909894d638ec3bd697f6ecb32(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:57,403 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:13:57,403 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/A, priority=13, startTime=1731975236768; duration=0sec 2024-11-19T00:13:57,403 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:57,403 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:A 2024-11-19T00:13:57,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741987_1163 (size=12151) 2024-11-19T00:13:57,412 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/319dd584a3d64c6abe9d09f70d78543c 2024-11-19T00:13:57,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975297434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:57,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/789c8c73feaf49e593286de256f9d8ab is 50, key is test_row_0/C:col10/1731975235382/Put/seqid=0 2024-11-19T00:13:57,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975297443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:57,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:57,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:57,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:57,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741988_1164 (size=12151) 2024-11-19T00:13:57,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:57,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47040 deadline: 1731975297554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:57,562 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_2' in 0f7b113264f1c67c501d9bd1480cc2b7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:13:57,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:57,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:57,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:57,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47048 deadline: 1731975297563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:57,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46992 deadline: 1731975297564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:57,569 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_0' in 0f7b113264f1c67c501d9bd1480cc2b7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:13:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:57,569 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_1' in 0f7b113264f1c67c501d9bd1480cc2b7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:13:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:57,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T00:13:57,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:57,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975297743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:57,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:57,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:57,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975297754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:57,888 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/789c8c73feaf49e593286de256f9d8ab 2024-11-19T00:13:57,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/779a9c63f4dc42afa0bf4766b5d50bda as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/779a9c63f4dc42afa0bf4766b5d50bda 2024-11-19T00:13:57,898 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/779a9c63f4dc42afa0bf4766b5d50bda, entries=150, sequenceid=180, filesize=30.4 K 2024-11-19T00:13:57,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/319dd584a3d64c6abe9d09f70d78543c as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/319dd584a3d64c6abe9d09f70d78543c 2024-11-19T00:13:57,904 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/319dd584a3d64c6abe9d09f70d78543c, entries=150, sequenceid=180, filesize=11.9 K 2024-11-19T00:13:57,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/789c8c73feaf49e593286de256f9d8ab as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/789c8c73feaf49e593286de256f9d8ab 2024-11-19T00:13:57,911 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/789c8c73feaf49e593286de256f9d8ab, entries=150, sequenceid=180, filesize=11.9 K 2024-11-19T00:13:57,912 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 0f7b113264f1c67c501d9bd1480cc2b7 in 1127ms, sequenceid=180, compaction requested=false 2024-11-19T00:13:57,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:13:57,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:57,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-19T00:13:57,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-19T00:13:57,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-19T00:13:57,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3700 sec 2024-11-19T00:13:57,918 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.3920 sec 2024-11-19T00:13:58,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:58,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f7b113264f1c67c501d9bd1480cc2b7 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-19T00:13:58,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=A 2024-11-19T00:13:58,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:58,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=B 2024-11-19T00:13:58,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:58,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=C 2024-11-19T00:13:58,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:13:58,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119985d61759d014cce805e72c87215a1a6_0f7b113264f1c67c501d9bd1480cc2b7 is 50, key is test_row_0/A:col10/1731975238255/Put/seqid=0 2024-11-19T00:13:58,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741989_1165 (size=14794) 2024-11-19T00:13:58,273 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975298271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:58,278 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119985d61759d014cce805e72c87215a1a6_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119985d61759d014cce805e72c87215a1a6_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:58,280 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/1406fcbbf9eb45d1a5782f302f41ce23, store: [table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:58,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/1406fcbbf9eb45d1a5782f302f41ce23 is 175, key is test_row_0/A:col10/1731975238255/Put/seqid=0 2024-11-19T00:13:58,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975298276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:58,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:58,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:58,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:58,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741990_1166 (size=39749) 2024-11-19T00:13:58,302 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/1406fcbbf9eb45d1a5782f302f41ce23 2024-11-19T00:13:58,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/d5f2ed0e7ec043c1bf4142f0dc0fdd1c is 50, key is test_row_0/B:col10/1731975238255/Put/seqid=0 2024-11-19T00:13:58,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741991_1167 (size=12151) 2024-11-19T00:13:58,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975298383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:58,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:58,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:58,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:58,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:58,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975298393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:58,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:58,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:58,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:58,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:58,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975298594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:58,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:58,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975298603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:58,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/d5f2ed0e7ec043c1bf4142f0dc0fdd1c 2024-11-19T00:13:58,739 INFO [master/6b1531871d35:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-19T00:13:58,739 INFO [master/6b1531871d35:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-19T00:13:58,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/d2db5b407d224e17b56091ee4ac2df69 is 50, key is test_row_0/C:col10/1731975238255/Put/seqid=0 2024-11-19T00:13:58,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741992_1168 (size=12151) 2024-11-19T00:13:58,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/d2db5b407d224e17b56091ee4ac2df69 2024-11-19T00:13:58,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/1406fcbbf9eb45d1a5782f302f41ce23 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/1406fcbbf9eb45d1a5782f302f41ce23 2024-11-19T00:13:58,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/1406fcbbf9eb45d1a5782f302f41ce23, entries=200, sequenceid=220, filesize=38.8 K 2024-11-19T00:13:58,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/d5f2ed0e7ec043c1bf4142f0dc0fdd1c as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/d5f2ed0e7ec043c1bf4142f0dc0fdd1c 2024-11-19T00:13:58,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/d5f2ed0e7ec043c1bf4142f0dc0fdd1c, entries=150, sequenceid=220, filesize=11.9 K 2024-11-19T00:13:58,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/d2db5b407d224e17b56091ee4ac2df69 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/d2db5b407d224e17b56091ee4ac2df69 2024-11-19T00:13:58,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/d2db5b407d224e17b56091ee4ac2df69, entries=150, sequenceid=220, filesize=11.9 K 2024-11-19T00:13:58,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 0f7b113264f1c67c501d9bd1480cc2b7 in 541ms, sequenceid=220, compaction requested=true 2024-11-19T00:13:58,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:13:58,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:13:58,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:58,797 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:58,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:13:58,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:58,797 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:58,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:13:58,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:58,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,799 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:58,799 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:58,799 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/B is initiating minor compaction (all files) 2024-11-19T00:13:58,799 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/A is initiating minor compaction (all files) 2024-11-19T00:13:58,799 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/B in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:58,799 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/A in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,800 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/b7c7b39909894d638ec3bd697f6ecb32, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/779a9c63f4dc42afa0bf4766b5d50bda, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/1406fcbbf9eb45d1a5782f302f41ce23] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=99.9 K 2024-11-19T00:13:58,800 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/79d8469c250141a6ae7be93af1115278, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/319dd584a3d64c6abe9d09f70d78543c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/d5f2ed0e7ec043c1bf4142f0dc0fdd1c] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=35.9 K 2024-11-19T00:13:58,800 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:58,800 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. files: [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/b7c7b39909894d638ec3bd697f6ecb32, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/779a9c63f4dc42afa0bf4766b5d50bda, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/1406fcbbf9eb45d1a5782f302f41ce23] 2024-11-19T00:13:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,800 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 79d8469c250141a6ae7be93af1115278, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731975234746 2024-11-19T00:13:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,801 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7c7b39909894d638ec3bd697f6ecb32, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731975234746 2024-11-19T00:13:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,802 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 779a9c63f4dc42afa0bf4766b5d50bda, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731975235382 2024-11-19T00:13:58,802 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 319dd584a3d64c6abe9d09f70d78543c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731975235382 2024-11-19T00:13:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,802 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting d5f2ed0e7ec043c1bf4142f0dc0fdd1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731975237038 2024-11-19T00:13:58,802 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1406fcbbf9eb45d1a5782f302f41ce23, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731975237038 2024-11-19T00:13:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,811 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:58,813 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#B#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:58,814 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/ad33f75fae884f768ce332a381a8a9f3 is 50, key is test_row_0/B:col10/1731975238255/Put/seqid=0 2024-11-19T00:13:58,814 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119289ce2876ef14a5bb80b2ae2c52320d2_0f7b113264f1c67c501d9bd1480cc2b7 store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,816 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119289ce2876ef14a5bb80b2ae2c52320d2_0f7b113264f1c67c501d9bd1480cc2b7, store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:58,816 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119289ce2876ef14a5bb80b2ae2c52320d2_0f7b113264f1c67c501d9bd1480cc2b7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:58,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741993_1169 (size=12561) 2024-11-19T00:13:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741994_1170 (size=4469) 2024-11-19T00:13:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,854 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/ad33f75fae884f768ce332a381a8a9f3 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ad33f75fae884f768ce332a381a8a9f3 2024-11-19T00:13:58,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,865 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/B of 0f7b113264f1c67c501d9bd1480cc2b7 into ad33f75fae884f768ce332a381a8a9f3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:58,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,865 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:13:58,865 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/B, priority=13, startTime=1731975238797; duration=0sec 2024-11-19T00:13:58,865 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:13:58,865 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:B 2024-11-19T00:13:58,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,866 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:13:58,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,868 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:13:58,868 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/C is initiating minor compaction (all files) 2024-11-19T00:13:58,869 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/C in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:58,869 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/e219cb785e914fb6bdd12a486db80abb, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/789c8c73feaf49e593286de256f9d8ab, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/d2db5b407d224e17b56091ee4ac2df69] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=35.9 K 2024-11-19T00:13:58,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,869 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting e219cb785e914fb6bdd12a486db80abb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731975234746 2024-11-19T00:13:58,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,870 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 789c8c73feaf49e593286de256f9d8ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731975235382 2024-11-19T00:13:58,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,870 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting d2db5b407d224e17b56091ee4ac2df69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731975237038 2024-11-19T00:13:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,884 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#C#compaction#143 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:58,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,885 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/d5e0f6ccf5814edf926cdc2c28958c1b is 50, key is test_row_0/C:col10/1731975238255/Put/seqid=0 2024-11-19T00:13:58,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741995_1171 (size=12561) 2024-11-19T00:13:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:58,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f7b113264f1c67c501d9bd1480cc2b7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T00:13:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=A 2024-11-19T00:13:58,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:58,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=B 2024-11-19T00:13:58,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:58,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=C 2024-11-19T00:13:58,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:13:58,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119606e3399c9a345b3b6ec55ee754f2d24_0f7b113264f1c67c501d9bd1480cc2b7 is 50, key is test_row_0/A:col10/1731975238267/Put/seqid=0 2024-11-19T00:13:58,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741996_1172 (size=14794) 2024-11-19T00:13:58,957 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:13:58,965 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119606e3399c9a345b3b6ec55ee754f2d24_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119606e3399c9a345b3b6ec55ee754f2d24_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:13:58,966 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/a008098e03a44b1081956c8a6094e1a0, store: [table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:13:58,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/a008098e03a44b1081956c8a6094e1a0 is 175, key is test_row_0/A:col10/1731975238267/Put/seqid=0 2024-11-19T00:13:58,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741997_1173 (size=39749) 2024-11-19T00:13:58,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975238960, max timestamp=1731975238990Num uniques 859; 2024-11-19T00:13:58,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:13:58,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975238960, max timestamp=1731975238990Num uniques 859; 2024-11-19T00:13:58,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975238960, max timestamp=1731975238990Num uniques 859; 2024-11-19T00:13:58,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:13:58,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:13:59,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:59,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975299058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:59,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:59,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:59,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975299059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:59,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:59,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:59,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:59,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:59,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:59,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:59,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975299174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:59,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:59,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975299174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:59,255 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#A#compaction#142 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:13:59,255 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/4a3fab8788604c6694d1d24b58db3a86 is 175, key is test_row_0/A:col10/1731975238255/Put/seqid=0 2024-11-19T00:13:59,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741998_1174 (size=31515) 2024-11-19T00:13:59,310 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/d5e0f6ccf5814edf926cdc2c28958c1b as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/d5e0f6ccf5814edf926cdc2c28958c1b 2024-11-19T00:13:59,316 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/C of 0f7b113264f1c67c501d9bd1480cc2b7 into d5e0f6ccf5814edf926cdc2c28958c1b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:59,316 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:13:59,316 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/C, priority=13, startTime=1731975238797; duration=0sec 2024-11-19T00:13:59,316 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:59,316 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:C 2024-11-19T00:13:59,371 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=231, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/a008098e03a44b1081956c8a6094e1a0 2024-11-19T00:13:59,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/ea9be7b1e9f644428041b63b93a8dd56 is 50, key is test_row_0/B:col10/1731975238267/Put/seqid=0 2024-11-19T00:13:59,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741999_1175 (size=12151) 2024-11-19T00:13:59,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975299383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:59,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975299383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:59,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:59,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:59,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:59,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:59,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:59,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47040 deadline: 1731975299574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:59,576 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_2' in 0f7b113264f1c67c501d9bd1480cc2b7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:13:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:59,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47048 deadline: 1731975299583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:59,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:59,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46992 deadline: 1731975299584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:59,587 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_0' in 0f7b113264f1c67c501d9bd1480cc2b7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:13:59,587 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_1' in 0f7b113264f1c67c501d9bd1480cc2b7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:13:59,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:59,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:59,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:59,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:59,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:59,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:59,665 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/4a3fab8788604c6694d1d24b58db3a86 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/4a3fab8788604c6694d1d24b58db3a86 2024-11-19T00:13:59,669 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/A of 0f7b113264f1c67c501d9bd1480cc2b7 into 4a3fab8788604c6694d1d24b58db3a86(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:13:59,669 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:13:59,669 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/A, priority=13, startTime=1731975238797; duration=0sec 2024-11-19T00:13:59,669 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:13:59,669 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:A 2024-11-19T00:13:59,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T00:13:59,694 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:13:59,694 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:13:59,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-19T00:13:59,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:59,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975299694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:59,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T00:13:59,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:59,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:59,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:59,697 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:13:59,697 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:13:59,698 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:13:59,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:13:59,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975299704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:13:59,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:59,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:13:59,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:13:59,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/ea9be7b1e9f644428041b63b93a8dd56 2024-11-19T00:13:59,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/72f3d71fd1994c4e8cbd5feb3832844a is 50, key is test_row_0/C:col10/1731975238267/Put/seqid=0 2024-11-19T00:13:59,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742000_1176 (size=12151) 2024-11-19T00:13:59,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T00:13:59,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T00:13:59,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:59,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:13:59,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:13:59,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:59,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:13:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:00,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T00:14:00,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:00,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:00,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:00,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:00,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:00,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T00:14:00,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T00:14:00,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:00,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:00,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:00,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:00,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:00,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/72f3d71fd1994c4e8cbd5feb3832844a 2024-11-19T00:14:00,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/a008098e03a44b1081956c8a6094e1a0 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/a008098e03a44b1081956c8a6094e1a0 2024-11-19T00:14:00,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:00,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975300204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:00,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:00,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/a008098e03a44b1081956c8a6094e1a0, entries=200, sequenceid=231, filesize=38.8 K 2024-11-19T00:14:00,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/ea9be7b1e9f644428041b63b93a8dd56 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ea9be7b1e9f644428041b63b93a8dd56 2024-11-19T00:14:00,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ea9be7b1e9f644428041b63b93a8dd56, entries=150, sequenceid=231, filesize=11.9 K 2024-11-19T00:14:00,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/72f3d71fd1994c4e8cbd5feb3832844a as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/72f3d71fd1994c4e8cbd5feb3832844a 2024-11-19T00:14:00,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/72f3d71fd1994c4e8cbd5feb3832844a, entries=150, sequenceid=231, filesize=11.9 K 2024-11-19T00:14:00,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0f7b113264f1c67c501d9bd1480cc2b7 in 1302ms, sequenceid=231, compaction requested=false 2024-11-19T00:14:00,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:00,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f7b113264f1c67c501d9bd1480cc2b7 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-19T00:14:00,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=A 2024-11-19T00:14:00,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:14:00,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=B 2024-11-19T00:14:00,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:14:00,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=C 2024-11-19T00:14:00,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:14:00,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119365c72462fe648ce95e94d1acc3a3079_0f7b113264f1c67c501d9bd1480cc2b7 is 50, key is test_row_0/A:col10/1731975239057/Put/seqid=0 2024-11-19T00:14:00,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742001_1177 (size=14994) 2024-11-19T00:14:00,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:00,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975300258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:00,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:00,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:00,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:00,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T00:14:00,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:00,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:00,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:00,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:00,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T00:14:00,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975300364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:00,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T00:14:00,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:00,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:00,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:00,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:00,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:00,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:00,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975300573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:00,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:00,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:00,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:00,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T00:14:00,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:00,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:00,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:00,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:00,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:00,653 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:00,657 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119365c72462fe648ce95e94d1acc3a3079_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119365c72462fe648ce95e94d1acc3a3079_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:00,658 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/40eda8f1f31742348f3897ddebba00a5, store: [table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:00,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/40eda8f1f31742348f3897ddebba00a5 is 175, key is test_row_0/A:col10/1731975239057/Put/seqid=0 2024-11-19T00:14:00,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742002_1178 (size=39949) 2024-11-19T00:14:00,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T00:14:00,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:00,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:00,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:00,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:00,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T00:14:00,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:00,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975300884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:00,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:00,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:00,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:00,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T00:14:00,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:00,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:00,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:00,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:00,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:00,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:01,071 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/40eda8f1f31742348f3897ddebba00a5 2024-11-19T00:14:01,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T00:14:01,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:01,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:01,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:01,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:01,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:01,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/3eb66e34398847bfb07699115c5bc751 is 50, key is test_row_0/B:col10/1731975239057/Put/seqid=0 2024-11-19T00:14:01,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742003_1179 (size=12301) 2024-11-19T00:14:01,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975301224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:01,226 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_0' in 0f7b113264f1c67c501d9bd1480cc2b7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:14:01,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:01,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:01,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:01,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T00:14:01,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:01,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:01,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:01,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:01,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:01,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T00:14:01,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:01,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:01,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:01,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:01,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:01,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975301393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:01,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/3eb66e34398847bfb07699115c5bc751 2024-11-19T00:14:01,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/f1c3311e36d248278d97bc6bff1d5f5a is 50, key is test_row_0/C:col10/1731975239057/Put/seqid=0 2024-11-19T00:14:01,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742004_1180 (size=12301) 2024-11-19T00:14:01,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T00:14:01,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:01,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:01,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:01,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:01,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:01,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T00:14:01,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:01,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:01,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:01,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:01,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:01,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:01,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T00:14:01,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:01,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:01,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:01,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:01,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:01,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T00:14:01,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:01,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/f1c3311e36d248278d97bc6bff1d5f5a 2024-11-19T00:14:01,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/40eda8f1f31742348f3897ddebba00a5 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/40eda8f1f31742348f3897ddebba00a5 2024-11-19T00:14:01,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/40eda8f1f31742348f3897ddebba00a5, entries=200, sequenceid=271, filesize=39.0 K 2024-11-19T00:14:01,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/3eb66e34398847bfb07699115c5bc751 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/3eb66e34398847bfb07699115c5bc751 2024-11-19T00:14:01,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/3eb66e34398847bfb07699115c5bc751, entries=150, sequenceid=271, filesize=12.0 K 2024-11-19T00:14:01,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/f1c3311e36d248278d97bc6bff1d5f5a as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/f1c3311e36d248278d97bc6bff1d5f5a 2024-11-19T00:14:01,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/f1c3311e36d248278d97bc6bff1d5f5a, entries=150, sequenceid=271, filesize=12.0 K 2024-11-19T00:14:01,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 0f7b113264f1c67c501d9bd1480cc2b7 in 1728ms, sequenceid=271, compaction requested=true 2024-11-19T00:14:01,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:01,956 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:01,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:14:01,958 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111213 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:01,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:01,958 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/A is initiating minor compaction (all files) 2024-11-19T00:14:01,958 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:01,958 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/A in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:01,958 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/4a3fab8788604c6694d1d24b58db3a86, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/a008098e03a44b1081956c8a6094e1a0, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/40eda8f1f31742348f3897ddebba00a5] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=108.6 K 2024-11-19T00:14:01,959 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:01,959 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. files: [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/4a3fab8788604c6694d1d24b58db3a86, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/a008098e03a44b1081956c8a6094e1a0, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/40eda8f1f31742348f3897ddebba00a5] 2024-11-19T00:14:01,960 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a3fab8788604c6694d1d24b58db3a86, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731975237038 2024-11-19T00:14:01,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:14:01,960 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting a008098e03a44b1081956c8a6094e1a0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731975238267 2024-11-19T00:14:01,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:01,960 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:01,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:14:01,961 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/B is initiating minor compaction (all files) 2024-11-19T00:14:01,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:01,961 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/B in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:01,961 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ad33f75fae884f768ce332a381a8a9f3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ea9be7b1e9f644428041b63b93a8dd56, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/3eb66e34398847bfb07699115c5bc751] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=36.1 K 2024-11-19T00:14:01,961 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40eda8f1f31742348f3897ddebba00a5, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731975239040 2024-11-19T00:14:01,964 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting ad33f75fae884f768ce332a381a8a9f3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731975237038 2024-11-19T00:14:01,965 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting ea9be7b1e9f644428041b63b93a8dd56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731975238267 2024-11-19T00:14:01,966 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eb66e34398847bfb07699115c5bc751, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731975239050 2024-11-19T00:14:01,991 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:01,996 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#B#compaction#151 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:01,997 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/7786291da16d4cf8886363d8b0506ad0 is 50, key is test_row_0/B:col10/1731975239057/Put/seqid=0 2024-11-19T00:14:01,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T00:14:01,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:01,998 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 0f7b113264f1c67c501d9bd1480cc2b7 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-19T00:14:01,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=A 2024-11-19T00:14:01,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:01,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=B 2024-11-19T00:14:01,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:01,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=C 2024-11-19T00:14:01,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:02,005 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119ae488fdee8924d85bb615487e0a634f7_0f7b113264f1c67c501d9bd1480cc2b7 store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:02,007 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119ae488fdee8924d85bb615487e0a634f7_0f7b113264f1c67c501d9bd1480cc2b7, store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:02,008 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ae488fdee8924d85bb615487e0a634f7_0f7b113264f1c67c501d9bd1480cc2b7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:02,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e8fe934c4ebd4596b402606f99eb00a6_0f7b113264f1c67c501d9bd1480cc2b7 is 50, key is test_row_0/A:col10/1731975240256/Put/seqid=0 2024-11-19T00:14:02,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742005_1181 (size=4469) 2024-11-19T00:14:02,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742006_1182 (size=12813) 2024-11-19T00:14:02,059 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#A#compaction#150 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:02,060 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/77f39d656d214d7f97355d3ae8913e90 is 175, key is test_row_0/A:col10/1731975239057/Put/seqid=0 2024-11-19T00:14:02,071 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/7786291da16d4cf8886363d8b0506ad0 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/7786291da16d4cf8886363d8b0506ad0 2024-11-19T00:14:02,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742007_1183 (size=12454) 2024-11-19T00:14:02,083 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/B of 0f7b113264f1c67c501d9bd1480cc2b7 into 7786291da16d4cf8886363d8b0506ad0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:02,083 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:02,083 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/B, priority=13, startTime=1731975241958; duration=0sec 2024-11-19T00:14:02,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:02,085 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:02,085 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:B 2024-11-19T00:14:02,085 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:02,087 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:02,088 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/C is initiating minor compaction (all files) 2024-11-19T00:14:02,088 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/C in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:02,089 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/d5e0f6ccf5814edf926cdc2c28958c1b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/72f3d71fd1994c4e8cbd5feb3832844a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/f1c3311e36d248278d97bc6bff1d5f5a] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=36.1 K 2024-11-19T00:14:02,089 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting d5e0f6ccf5814edf926cdc2c28958c1b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731975237038 2024-11-19T00:14:02,090 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e8fe934c4ebd4596b402606f99eb00a6_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e8fe934c4ebd4596b402606f99eb00a6_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:02,093 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 72f3d71fd1994c4e8cbd5feb3832844a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731975238267 2024-11-19T00:14:02,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/e3f1c44626284d36a55bebf9a552950f, store: [table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:02,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/e3f1c44626284d36a55bebf9a552950f is 175, key is test_row_0/A:col10/1731975240256/Put/seqid=0 2024-11-19T00:14:02,094 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting f1c3311e36d248278d97bc6bff1d5f5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731975239050 2024-11-19T00:14:02,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742008_1184 (size=31767) 2024-11-19T00:14:02,121 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/77f39d656d214d7f97355d3ae8913e90 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/77f39d656d214d7f97355d3ae8913e90 2024-11-19T00:14:02,129 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#C#compaction#153 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:02,129 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/A of 0f7b113264f1c67c501d9bd1480cc2b7 into 77f39d656d214d7f97355d3ae8913e90(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:02,129 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:02,129 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/A, priority=13, startTime=1731975241956; duration=0sec 2024-11-19T00:14:02,129 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/1adc1db07fb34e479a2482988c286795 is 50, key is test_row_0/C:col10/1731975239057/Put/seqid=0 2024-11-19T00:14:02,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742009_1185 (size=31255) 2024-11-19T00:14:02,131 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:02,132 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:A 2024-11-19T00:14:02,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742010_1186 (size=12813) 2024-11-19T00:14:02,162 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/1adc1db07fb34e479a2482988c286795 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/1adc1db07fb34e479a2482988c286795 2024-11-19T00:14:02,176 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/C of 0f7b113264f1c67c501d9bd1480cc2b7 into 1adc1db07fb34e479a2482988c286795(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:02,176 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:02,176 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/C, priority=13, startTime=1731975241960; duration=0sec 2024-11-19T00:14:02,176 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:02,176 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:C 2024-11-19T00:14:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:02,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:02,533 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/e3f1c44626284d36a55bebf9a552950f 2024-11-19T00:14:02,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/a05c083fac2a46c38837bf3e01bc2753 is 50, key is test_row_0/B:col10/1731975240256/Put/seqid=0 2024-11-19T00:14:02,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975242460, max timestamp=1731975242543Num uniques 859; 2024-11-19T00:14:02,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975242460, max timestamp=1731975242543Num uniques 859; 2024-11-19T00:14:02,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:14:02,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975242460, max timestamp=1731975242543Num uniques 859; 2024-11-19T00:14:02,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:14:02,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:14:02,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742011_1187 (size=12301) 2024-11-19T00:14:02,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731975242626, max timestamp=1731975242650Num uniques 1018; 2024-11-19T00:14:02,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T00:14:02,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731975242626, max timestamp=1731975242650Num uniques 1018; 2024-11-19T00:14:02,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T00:14:02,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731975242626, max timestamp=1731975242650Num uniques 1018; 2024-11-19T00:14:02,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T00:14:02,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731975242731, max timestamp=1731975242770Num uniques 1027; 2024-11-19T00:14:02,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-19T00:14:02,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731975242731, max timestamp=1731975242770Num uniques 1027; 2024-11-19T00:14:02,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-19T00:14:02,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731975242731, max timestamp=1731975242770Num uniques 1027; 2024-11-19T00:14:02,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-19T00:14:02,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:02,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975302879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:02,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:02,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:02,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:02,980 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/a05c083fac2a46c38837bf3e01bc2753 2024-11-19T00:14:03,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:03,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975302999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:03,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:03,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:03,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:03,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/386e63deb2e14dfaa92d24e91127b135 is 50, key is test_row_0/C:col10/1731975240256/Put/seqid=0 2024-11-19T00:14:03,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742012_1188 (size=12301) 2024-11-19T00:14:03,046 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/386e63deb2e14dfaa92d24e91127b135 2024-11-19T00:14:03,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/e3f1c44626284d36a55bebf9a552950f as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/e3f1c44626284d36a55bebf9a552950f 2024-11-19T00:14:03,068 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/e3f1c44626284d36a55bebf9a552950f, entries=150, sequenceid=280, filesize=30.5 K 2024-11-19T00:14:03,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/a05c083fac2a46c38837bf3e01bc2753 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/a05c083fac2a46c38837bf3e01bc2753 2024-11-19T00:14:03,082 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/a05c083fac2a46c38837bf3e01bc2753, entries=150, sequenceid=280, filesize=12.0 K 2024-11-19T00:14:03,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/386e63deb2e14dfaa92d24e91127b135 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/386e63deb2e14dfaa92d24e91127b135 2024-11-19T00:14:03,090 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/386e63deb2e14dfaa92d24e91127b135, entries=150, sequenceid=280, filesize=12.0 K 2024-11-19T00:14:03,091 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 0f7b113264f1c67c501d9bd1480cc2b7 in 1093ms, sequenceid=280, compaction requested=false 2024-11-19T00:14:03,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:03,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:03,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-19T00:14:03,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-19T00:14:03,095 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-19T00:14:03,095 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3950 sec 2024-11-19T00:14:03,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 3.4020 sec 2024-11-19T00:14:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:03,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f7b113264f1c67c501d9bd1480cc2b7 3/3 column families, dataSize=147.60 KB heapSize=419.16 KB 2024-11-19T00:14:03,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=A 2024-11-19T00:14:03,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:14:03,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=B 2024-11-19T00:14:03,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:14:03,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=C 2024-11-19T00:14:03,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:14:03,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:03,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975303238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:03,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:03,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:03,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:03,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111960d1331504924baebe628f28e72b5375_0f7b113264f1c67c501d9bd1480cc2b7 is 50, key is test_row_0/A:col10/1731975242820/Put/seqid=0 2024-11-19T00:14:03,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975303253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:03,258 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_0' in 0f7b113264f1c67c501d9bd1480cc2b7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:14:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:03,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742013_1189 (size=14994) 2024-11-19T00:14:03,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975303354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:03,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:03,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:03,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:03,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:03,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975303564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:03,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:03,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47040 deadline: 1731975303594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:03,603 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_2' in 0f7b113264f1c67c501d9bd1480cc2b7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:14:03,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:03,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:03,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:03,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46992 deadline: 1731975303603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:03,604 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_1' in 0f7b113264f1c67c501d9bd1480cc2b7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:14:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:03,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:03,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47048 deadline: 1731975303624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:03,628 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_0' in 0f7b113264f1c67c501d9bd1480cc2b7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:14:03,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:03,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:03,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:03,681 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:03,685 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111960d1331504924baebe628f28e72b5375_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111960d1331504924baebe628f28e72b5375_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:03,689 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/fde9cda241b044d282e68bb587991c03, store: [table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:03,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/fde9cda241b044d282e68bb587991c03 is 175, key is test_row_0/A:col10/1731975242820/Put/seqid=0 2024-11-19T00:14:03,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742014_1190 (size=39949) 2024-11-19T00:14:03,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T00:14:03,865 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:14:03,865 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:14:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-19T00:14:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T00:14:03,868 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:14:03,868 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:14:03,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:14:03,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:03,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975303873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:03,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T00:14:04,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T00:14:04,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:04,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:04,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:04,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:04,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:04,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:04,106 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=355, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/fde9cda241b044d282e68bb587991c03 2024-11-19T00:14:04,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/2957713b73b14b58a2e766f21a23b984 is 50, key is test_row_0/B:col10/1731975242820/Put/seqid=0 2024-11-19T00:14:04,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742015_1191 (size=12301) 2024-11-19T00:14:04,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T00:14:04,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:04,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:04,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:04,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:04,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:04,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T00:14:04,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T00:14:04,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:04,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:04,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:04,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:04,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:04,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:04,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:04,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975304384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:04,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:04,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T00:14:04,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:04,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:04,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:04,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:04,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:04,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:04,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T00:14:04,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/2957713b73b14b58a2e766f21a23b984 2024-11-19T00:14:04,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/1c3b0307178f4b9897368e737a0d5e02 is 50, key is test_row_0/C:col10/1731975242820/Put/seqid=0 2024-11-19T00:14:04,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742016_1192 (size=12301) 2024-11-19T00:14:04,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T00:14:04,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:04,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:04,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:04,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:04,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:04,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:04,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T00:14:04,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:04,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:04,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:04,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:04,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:04,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:04,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T00:14:04,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:04,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:04,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:04,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:04,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:04,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:04,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/1c3b0307178f4b9897368e737a0d5e02 2024-11-19T00:14:04,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/fde9cda241b044d282e68bb587991c03 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/fde9cda241b044d282e68bb587991c03 2024-11-19T00:14:04,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/fde9cda241b044d282e68bb587991c03, entries=200, sequenceid=355, filesize=39.0 K 2024-11-19T00:14:04,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/2957713b73b14b58a2e766f21a23b984 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/2957713b73b14b58a2e766f21a23b984 2024-11-19T00:14:04,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/2957713b73b14b58a2e766f21a23b984, entries=150, sequenceid=355, filesize=12.0 K 2024-11-19T00:14:04,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/1c3b0307178f4b9897368e737a0d5e02 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/1c3b0307178f4b9897368e737a0d5e02 2024-11-19T00:14:04,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/1c3b0307178f4b9897368e737a0d5e02, entries=150, sequenceid=355, filesize=12.0 K 2024-11-19T00:14:04,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.11 KB/429168, currentSize=33.54 KB/34350 for 0f7b113264f1c67c501d9bd1480cc2b7 in 1754ms, sequenceid=355, compaction requested=true 2024-11-19T00:14:04,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:04,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:14:04,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:04,963 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:04,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:14:04,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:04,963 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:04,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:14:04,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:04,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,964 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:04,964 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:04,964 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/A is initiating minor compaction (all files) 2024-11-19T00:14:04,964 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/B is initiating minor compaction (all files) 2024-11-19T00:14:04,964 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/A in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:04,964 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/B in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:04,965 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/7786291da16d4cf8886363d8b0506ad0, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/a05c083fac2a46c38837bf3e01bc2753, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/2957713b73b14b58a2e766f21a23b984] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=36.5 K 2024-11-19T00:14:04,965 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/77f39d656d214d7f97355d3ae8913e90, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/e3f1c44626284d36a55bebf9a552950f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/fde9cda241b044d282e68bb587991c03] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=100.6 K 2024-11-19T00:14:04,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,965 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:04,965 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. files: [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/77f39d656d214d7f97355d3ae8913e90, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/e3f1c44626284d36a55bebf9a552950f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/fde9cda241b044d282e68bb587991c03] 2024-11-19T00:14:04,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,966 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77f39d656d214d7f97355d3ae8913e90, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731975239050 2024-11-19T00:14:04,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,966 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3f1c44626284d36a55bebf9a552950f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731975240245 2024-11-19T00:14:04,966 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 7786291da16d4cf8886363d8b0506ad0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731975239050 2024-11-19T00:14:04,966 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting fde9cda241b044d282e68bb587991c03, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731975242820 2024-11-19T00:14:04,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,966 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting a05c083fac2a46c38837bf3e01bc2753, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731975240245 2024-11-19T00:14:04,967 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 2957713b73b14b58a2e766f21a23b984, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731975242820 2024-11-19T00:14:04,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,974 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:04,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,975 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#B#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:04,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,976 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/5c52ef5cc2414f11babd3e9740499441 is 50, key is test_row_0/B:col10/1731975242820/Put/seqid=0 2024-11-19T00:14:04,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,977 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119c9c1aa705bc940e0be51819264922af7_0f7b113264f1c67c501d9bd1480cc2b7 store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:04,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,980 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119c9c1aa705bc940e0be51819264922af7_0f7b113264f1c67c501d9bd1480cc2b7, store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:04,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,980 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c9c1aa705bc940e0be51819264922af7_0f7b113264f1c67c501d9bd1480cc2b7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:04,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742017_1193 (size=12915) 2024-11-19T00:14:04,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742018_1194 (size=4469) 2024-11-19T00:14:04,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,997 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#A#compaction#160 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:04,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,997 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/4527823ca32d4f88aa175ebc6a9ad175 is 175, key is test_row_0/A:col10/1731975242820/Put/seqid=0 2024-11-19T00:14:04,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:04,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,002 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/5c52ef5cc2414f11babd3e9740499441 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/5c52ef5cc2414f11babd3e9740499441 2024-11-19T00:14:05,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T00:14:05,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,011 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/B of 0f7b113264f1c67c501d9bd1480cc2b7 into 5c52ef5cc2414f11babd3e9740499441(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:05,011 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:05,011 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/B, priority=13, startTime=1731975244963; duration=0sec 2024-11-19T00:14:05,011 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:05,011 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:B 2024-11-19T00:14:05,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,011 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:05,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742019_1195 (size=31869) 2024-11-19T00:14:05,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,013 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:05,013 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/C is initiating minor compaction (all files) 2024-11-19T00:14:05,013 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/C in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:05,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,014 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/1adc1db07fb34e479a2482988c286795, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/386e63deb2e14dfaa92d24e91127b135, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/1c3b0307178f4b9897368e737a0d5e02] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=36.5 K 2024-11-19T00:14:05,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,014 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 1adc1db07fb34e479a2482988c286795, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731975239050 2024-11-19T00:14:05,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,015 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 386e63deb2e14dfaa92d24e91127b135, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731975240245 2024-11-19T00:14:05,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,015 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c3b0307178f4b9897368e737a0d5e02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731975242820 2024-11-19T00:14:05,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,018 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/4527823ca32d4f88aa175ebc6a9ad175 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/4527823ca32d4f88aa175ebc6a9ad175 2024-11-19T00:14:05,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,024 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/A of 0f7b113264f1c67c501d9bd1480cc2b7 into 4527823ca32d4f88aa175ebc6a9ad175(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:05,024 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:05,024 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/A, priority=13, startTime=1731975244962; duration=0sec 2024-11-19T00:14:05,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,024 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:05,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,024 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:A 2024-11-19T00:14:05,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,026 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#C#compaction#161 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:05,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,026 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/cfdb95a0317e4129bb7f363ddd70b359 is 50, key is test_row_0/C:col10/1731975242820/Put/seqid=0 2024-11-19T00:14:05,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742020_1196 (size=12915) 2024-11-19T00:14:05,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,039 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/cfdb95a0317e4129bb7f363ddd70b359 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/cfdb95a0317e4129bb7f363ddd70b359 2024-11-19T00:14:05,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,045 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/C of 0f7b113264f1c67c501d9bd1480cc2b7 into cfdb95a0317e4129bb7f363ddd70b359(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:05,045 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:05,045 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/C, priority=13, startTime=1731975244963; duration=0sec 2024-11-19T00:14:05,046 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:05,046 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:C 2024-11-19T00:14:05,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T00:14:05,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:05,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,092 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 0f7b113264f1c67c501d9bd1480cc2b7 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T00:14:05,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=A 2024-11-19T00:14:05,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:05,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=B 2024-11-19T00:14:05,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:05,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=C 2024-11-19T00:14:05,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:05,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119643955bc91e542c7ad61e87a6155de28_0f7b113264f1c67c501d9bd1480cc2b7 is 50, key is test_row_0/A:col10/1731975243235/Put/seqid=0 2024-11-19T00:14:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742021_1197 (size=9914) 2024-11-19T00:14:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:05,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975245470, max timestamp=1731975245484Num uniques 859; 2024-11-19T00:14:05,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:14:05,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975245470, max timestamp=1731975245484Num uniques 859; 2024-11-19T00:14:05,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:14:05,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975245470, max timestamp=1731975245484Num uniques 859; 2024-11-19T00:14:05,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:14:05,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:05,529 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119643955bc91e542c7ad61e87a6155de28_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119643955bc91e542c7ad61e87a6155de28_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:05,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/23481213c39f498cb37051a92e6c2c6d, store: [table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:05,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/23481213c39f498cb37051a92e6c2c6d is 175, key is test_row_0/A:col10/1731975243235/Put/seqid=0 2024-11-19T00:14:05,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742022_1198 (size=22561) 2024-11-19T00:14:05,541 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=366, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/23481213c39f498cb37051a92e6c2c6d 2024-11-19T00:14:05,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/d5ab497f8aec413c9c2c8746eea9f383 is 50, key is test_row_0/B:col10/1731975243235/Put/seqid=0 2024-11-19T00:14:05,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742023_1199 (size=9857) 2024-11-19T00:14:05,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731975245524, max timestamp=1731975245559Num uniques 1018; 2024-11-19T00:14:05,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T00:14:05,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731975245524, max timestamp=1731975245559Num uniques 1018; 2024-11-19T00:14:05,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731975245524, max timestamp=1731975245559Num uniques 1018; 2024-11-19T00:14:05,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T00:14:05,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T00:14:05,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731975245598, max timestamp=1731975245619Num uniques 1077; 2024-11-19T00:14:05,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731975245598, max timestamp=1731975245619Num uniques 1077; 2024-11-19T00:14:05,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731975245598, max timestamp=1731975245619Num uniques 1077; 2024-11-19T00:14:05,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-19T00:14:05,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-19T00:14:05,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-19T00:14:05,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731975245660, max timestamp=1731975245678Num uniques 1086; 2024-11-19T00:14:05,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-19T00:14:05,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731975245660, max timestamp=1731975245678Num uniques 1086; 2024-11-19T00:14:05,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-19T00:14:05,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731975245660, max timestamp=1731975245678Num uniques 1086; 2024-11-19T00:14:05,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-19T00:14:05,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731975245715, max timestamp=1731975245735Num uniques 1095; 2024-11-19T00:14:05,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-19T00:14:05,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731975245715, max timestamp=1731975245735Num uniques 1095; 2024-11-19T00:14:05,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-19T00:14:05,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731975245715, max timestamp=1731975245735Num uniques 1095; 2024-11-19T00:14:05,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-19T00:14:05,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:05,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975305818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:05,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975305924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:05,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:05,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:05,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:05,965 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/d5ab497f8aec413c9c2c8746eea9f383 2024-11-19T00:14:05,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/e35fe3d84fbc4e9ebba48db4767957ec is 50, key is test_row_0/C:col10/1731975243235/Put/seqid=0 2024-11-19T00:14:05,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742024_1200 (size=9857) 2024-11-19T00:14:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T00:14:06,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975306134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:06,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:06,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:06,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:06,378 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/e35fe3d84fbc4e9ebba48db4767957ec 2024-11-19T00:14:06,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/23481213c39f498cb37051a92e6c2c6d as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/23481213c39f498cb37051a92e6c2c6d 2024-11-19T00:14:06,387 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/23481213c39f498cb37051a92e6c2c6d, entries=100, sequenceid=366, filesize=22.0 K 2024-11-19T00:14:06,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/d5ab497f8aec413c9c2c8746eea9f383 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/d5ab497f8aec413c9c2c8746eea9f383 2024-11-19T00:14:06,392 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/d5ab497f8aec413c9c2c8746eea9f383, entries=100, sequenceid=366, filesize=9.6 K 2024-11-19T00:14:06,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/e35fe3d84fbc4e9ebba48db4767957ec as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/e35fe3d84fbc4e9ebba48db4767957ec 2024-11-19T00:14:06,398 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/e35fe3d84fbc4e9ebba48db4767957ec, entries=100, sequenceid=366, filesize=9.6 K 2024-11-19T00:14:06,399 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 0f7b113264f1c67c501d9bd1480cc2b7 in 1308ms, sequenceid=366, compaction requested=false 2024-11-19T00:14:06,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:06,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:06,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-19T00:14:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-19T00:14:06,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-19T00:14:06,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5330 sec 2024-11-19T00:14:06,404 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.5380 sec 2024-11-19T00:14:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:06,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f7b113264f1c67c501d9bd1480cc2b7 3/3 column families, dataSize=154.31 KB heapSize=437.39 KB 2024-11-19T00:14:06,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=A 2024-11-19T00:14:06,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:14:06,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=B 2024-11-19T00:14:06,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:14:06,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=C 2024-11-19T00:14:06,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:14:06,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a8d283b30fbb47599cdf9bbffc8be1f7_0f7b113264f1c67c501d9bd1480cc2b7 is 50, key is test_row_0/A:col10/1731975245816/Put/seqid=0 2024-11-19T00:14:06,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742025_1201 (size=14994) 2024-11-19T00:14:06,463 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:06,467 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a8d283b30fbb47599cdf9bbffc8be1f7_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a8d283b30fbb47599cdf9bbffc8be1f7_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:06,468 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/43ca107fb81d4336a1a57478c00d5831, store: [table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:06,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/43ca107fb81d4336a1a57478c00d5831 is 175, key is test_row_0/A:col10/1731975245816/Put/seqid=0 2024-11-19T00:14:06,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742026_1202 (size=39949) 2024-11-19T00:14:06,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 295 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975306470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:06,475 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=473, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/43ca107fb81d4336a1a57478c00d5831 2024-11-19T00:14:06,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:06,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:06,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:06,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/756e26d9317d441eae4dbe3ffc9c8803 is 50, key is test_row_0/B:col10/1731975245816/Put/seqid=0 2024-11-19T00:14:06,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742027_1203 (size=12301) 2024-11-19T00:14:06,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/756e26d9317d441eae4dbe3ffc9c8803 2024-11-19T00:14:06,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/5e9f6033d53c4786a042261e6d09c34e is 50, key is test_row_0/C:col10/1731975245816/Put/seqid=0 2024-11-19T00:14:06,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742028_1204 (size=12301) 2024-11-19T00:14:06,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 296 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975306583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:06,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:06,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:06,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:06,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 297 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975306793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:06,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:06,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:06,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:06,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/5e9f6033d53c4786a042261e6d09c34e 2024-11-19T00:14:06,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/43ca107fb81d4336a1a57478c00d5831 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/43ca107fb81d4336a1a57478c00d5831 2024-11-19T00:14:06,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/43ca107fb81d4336a1a57478c00d5831, entries=200, sequenceid=473, filesize=39.0 K 2024-11-19T00:14:06,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/756e26d9317d441eae4dbe3ffc9c8803 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/756e26d9317d441eae4dbe3ffc9c8803 2024-11-19T00:14:06,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/756e26d9317d441eae4dbe3ffc9c8803, entries=150, sequenceid=473, filesize=12.0 K 2024-11-19T00:14:06,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/5e9f6033d53c4786a042261e6d09c34e as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/5e9f6033d53c4786a042261e6d09c34e 2024-11-19T00:14:06,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/5e9f6033d53c4786a042261e6d09c34e, entries=150, sequenceid=473, filesize=12.0 K 2024-11-19T00:14:06,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~437.34 KB/447840, currentSize=26.84 KB/27480 for 0f7b113264f1c67c501d9bd1480cc2b7 in 502ms, sequenceid=473, compaction requested=true 2024-11-19T00:14:06,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:06,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:14:06,950 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:06,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:06,950 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:06,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:14:06,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:06,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:14:06,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:06,951 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:06,951 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:06,951 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/A is initiating minor compaction (all files) 2024-11-19T00:14:06,951 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/B is initiating minor compaction (all files) 2024-11-19T00:14:06,951 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/B in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:06,951 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/A in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:06,951 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/5c52ef5cc2414f11babd3e9740499441, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/d5ab497f8aec413c9c2c8746eea9f383, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/756e26d9317d441eae4dbe3ffc9c8803] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=34.3 K 2024-11-19T00:14:06,951 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/4527823ca32d4f88aa175ebc6a9ad175, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/23481213c39f498cb37051a92e6c2c6d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/43ca107fb81d4336a1a57478c00d5831] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=92.2 K 2024-11-19T00:14:06,951 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:06,951 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. files: [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/4527823ca32d4f88aa175ebc6a9ad175, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/23481213c39f498cb37051a92e6c2c6d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/43ca107fb81d4336a1a57478c00d5831] 2024-11-19T00:14:06,951 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c52ef5cc2414f11babd3e9740499441, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731975242820 2024-11-19T00:14:06,951 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4527823ca32d4f88aa175ebc6a9ad175, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731975242820 2024-11-19T00:14:06,952 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting d5ab497f8aec413c9c2c8746eea9f383, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731975243224 2024-11-19T00:14:06,952 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23481213c39f498cb37051a92e6c2c6d, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731975243224 2024-11-19T00:14:06,952 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 756e26d9317d441eae4dbe3ffc9c8803, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731975245780 2024-11-19T00:14:06,952 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43ca107fb81d4336a1a57478c00d5831, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731975245780 2024-11-19T00:14:06,959 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:06,961 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411198a16903c0437411fa8055f9eb76e2d81_0f7b113264f1c67c501d9bd1480cc2b7 store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:06,962 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#B#compaction#169 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:06,962 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/e0e89d7148ef4fada961dacb2a468322 is 50, key is test_row_0/B:col10/1731975245816/Put/seqid=0 2024-11-19T00:14:06,963 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411198a16903c0437411fa8055f9eb76e2d81_0f7b113264f1c67c501d9bd1480cc2b7, store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:06,964 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198a16903c0437411fa8055f9eb76e2d81_0f7b113264f1c67c501d9bd1480cc2b7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:06,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742029_1205 (size=4469) 2024-11-19T00:14:06,975 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#A#compaction#168 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:06,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742030_1206 (size=13017) 2024-11-19T00:14:06,976 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/44569da080414525900941f8dd96a33b is 175, key is test_row_0/A:col10/1731975245816/Put/seqid=0 2024-11-19T00:14:06,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742031_1207 (size=31971) 2024-11-19T00:14:06,984 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/e0e89d7148ef4fada961dacb2a468322 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/e0e89d7148ef4fada961dacb2a468322 2024-11-19T00:14:06,989 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/B of 0f7b113264f1c67c501d9bd1480cc2b7 into e0e89d7148ef4fada961dacb2a468322(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:06,989 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:06,989 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/B, priority=13, startTime=1731975246950; duration=0sec 2024-11-19T00:14:06,989 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:06,989 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:B 2024-11-19T00:14:06,989 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:06,990 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:06,990 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/C is initiating minor compaction (all files) 2024-11-19T00:14:06,990 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/C in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:06,990 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/cfdb95a0317e4129bb7f363ddd70b359, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/e35fe3d84fbc4e9ebba48db4767957ec, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/5e9f6033d53c4786a042261e6d09c34e] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=34.3 K 2024-11-19T00:14:06,991 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting cfdb95a0317e4129bb7f363ddd70b359, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731975242820 2024-11-19T00:14:06,991 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting e35fe3d84fbc4e9ebba48db4767957ec, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731975243224 2024-11-19T00:14:06,991 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e9f6033d53c4786a042261e6d09c34e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731975245780 2024-11-19T00:14:07,000 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#C#compaction#170 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:07,001 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/8107ea3e066242aa8ea17b438b062d56 is 50, key is test_row_0/C:col10/1731975245816/Put/seqid=0 2024-11-19T00:14:07,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742032_1208 (size=13017) 2024-11-19T00:14:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:07,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f7b113264f1c67c501d9bd1480cc2b7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T00:14:07,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=A 2024-11-19T00:14:07,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:07,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=B 2024-11-19T00:14:07,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:07,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=C 2024-11-19T00:14:07,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:07,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119522ee531a7a44e1e98d745ac8d54eecc_0f7b113264f1c67c501d9bd1480cc2b7 is 50, key is test_row_0/A:col10/1731975247107/Put/seqid=0 2024-11-19T00:14:07,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742033_1209 (size=14994) 2024-11-19T00:14:07,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975247144, max timestamp=1731975247190Num uniques 859; 2024-11-19T00:14:07,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975247144, max timestamp=1731975247190Num uniques 859; 2024-11-19T00:14:07,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975247144, max timestamp=1731975247190Num uniques 859; 2024-11-19T00:14:07,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:14:07,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:14:07,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:14:07,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47050 deadline: 1731975307273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:07,278 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_0' in 0f7b113264f1c67c501d9bd1480cc2b7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:14:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:07,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 334 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975307278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:07,387 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/44569da080414525900941f8dd96a33b as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/44569da080414525900941f8dd96a33b 2024-11-19T00:14:07,391 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/A of 0f7b113264f1c67c501d9bd1480cc2b7 into 44569da080414525900941f8dd96a33b(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:07,391 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:07,391 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/A, priority=13, startTime=1731975246949; duration=0sec 2024-11-19T00:14:07,391 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:07,391 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:A 2024-11-19T00:14:07,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 335 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975307393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:07,410 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/8107ea3e066242aa8ea17b438b062d56 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/8107ea3e066242aa8ea17b438b062d56 2024-11-19T00:14:07,416 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/C of 0f7b113264f1c67c501d9bd1480cc2b7 into 8107ea3e066242aa8ea17b438b062d56(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:07,416 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:07,416 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/C, priority=13, startTime=1731975246950; duration=0sec 2024-11-19T00:14:07,416 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:07,416 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:C 2024-11-19T00:14:07,525 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:07,529 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119522ee531a7a44e1e98d745ac8d54eecc_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119522ee531a7a44e1e98d745ac8d54eecc_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:07,530 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/e5ddf727c55a4c94a5c6b2d14525c281, store: [table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:07,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/e5ddf727c55a4c94a5c6b2d14525c281 is 175, key is test_row_0/A:col10/1731975247107/Put/seqid=0 2024-11-19T00:14:07,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742034_1210 (size=39949) 2024-11-19T00:14:07,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 336 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975307604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:07,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:07,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:07,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:07,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:07,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 337 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975307914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:07,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:07,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:07,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:07,944 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=484, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/e5ddf727c55a4c94a5c6b2d14525c281 2024-11-19T00:14:07,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/e575a11a05c24d96abeff238ea096a1e is 50, key is test_row_0/B:col10/1731975247107/Put/seqid=0 2024-11-19T00:14:07,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742035_1211 (size=12301) 2024-11-19T00:14:07,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/e575a11a05c24d96abeff238ea096a1e 2024-11-19T00:14:07,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/b51b17b0096844ff8ae1ad5975f551c3 is 50, key is test_row_0/C:col10/1731975247107/Put/seqid=0 2024-11-19T00:14:07,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742036_1212 (size=12301) 2024-11-19T00:14:08,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T00:14:08,024 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:14:08,024 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:14:08,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-19T00:14:08,026 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:14:08,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T00:14:08,027 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:14:08,027 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:14:08,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T00:14:08,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T00:14:08,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:08,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:08,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:08,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:08,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:08,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:08,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T00:14:08,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:08,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:08,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:08,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:08,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:08,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T00:14:08,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/b51b17b0096844ff8ae1ad5975f551c3 2024-11-19T00:14:08,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/e5ddf727c55a4c94a5c6b2d14525c281 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/e5ddf727c55a4c94a5c6b2d14525c281 2024-11-19T00:14:08,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/e5ddf727c55a4c94a5c6b2d14525c281, entries=200, sequenceid=484, filesize=39.0 K 2024-11-19T00:14:08,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/e575a11a05c24d96abeff238ea096a1e as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/e575a11a05c24d96abeff238ea096a1e 2024-11-19T00:14:08,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/e575a11a05c24d96abeff238ea096a1e, entries=150, sequenceid=484, filesize=12.0 K 2024-11-19T00:14:08,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/b51b17b0096844ff8ae1ad5975f551c3 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/b51b17b0096844ff8ae1ad5975f551c3 2024-11-19T00:14:08,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/b51b17b0096844ff8ae1ad5975f551c3, entries=150, sequenceid=484, filesize=12.0 K 2024-11-19T00:14:08,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0f7b113264f1c67c501d9bd1480cc2b7 in 1279ms, sequenceid=484, compaction requested=false 2024-11-19T00:14:08,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:08,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f7b113264f1c67c501d9bd1480cc2b7 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-19T00:14:08,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=A 2024-11-19T00:14:08,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:14:08,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=B 2024-11-19T00:14:08,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:14:08,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=C 2024-11-19T00:14:08,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:14:08,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a0b859d6fc1a4db1953129f32ae8ece7_0f7b113264f1c67c501d9bd1480cc2b7 is 50, key is test_row_0/A:col10/1731975247226/Put/seqid=0 2024-11-19T00:14:08,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742037_1213 (size=14994) 2024-11-19T00:14:08,441 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:08,448 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a0b859d6fc1a4db1953129f32ae8ece7_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a0b859d6fc1a4db1953129f32ae8ece7_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:08,449 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/e1a5ff7ea72546aaaff4ec3199b49290, store: [table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:08,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/e1a5ff7ea72546aaaff4ec3199b49290 is 175, key is test_row_0/A:col10/1731975247226/Put/seqid=0 2024-11-19T00:14:08,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742038_1214 (size=39949) 2024-11-19T00:14:08,454 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=523, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/e1a5ff7ea72546aaaff4ec3199b49290 2024-11-19T00:14:08,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:08,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 345 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975308458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:08,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:08,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:08,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:08,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/be71b6655bcf46e8ae96860a99c40c6a is 50, key is test_row_0/B:col10/1731975247226/Put/seqid=0 2024-11-19T00:14:08,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742039_1215 (size=12301) 2024-11-19T00:14:08,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T00:14:08,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:08,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:08,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:08,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:08,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:08,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:08,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:08,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 346 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975308574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:08,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:08,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:08,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:08,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T00:14:08,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:08,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:08,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:08,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:08,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:08,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:08,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T00:14:08,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:08,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 347 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975308784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:08,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:08,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:08,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:08,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T00:14:08,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:08,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:08,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:08,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:08,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:08,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:08,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/be71b6655bcf46e8ae96860a99c40c6a 2024-11-19T00:14:08,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/76c31a2b1d3c4c32a41d6419f4fde9a0 is 50, key is test_row_0/C:col10/1731975247226/Put/seqid=0 2024-11-19T00:14:08,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742040_1216 (size=12301) 2024-11-19T00:14:08,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T00:14:08,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:08,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:08,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:08,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:08,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:09,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 348 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975309094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:09,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T00:14:09,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:09,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:09,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:09,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:09,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T00:14:09,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T00:14:09,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:09,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:09,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:09,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:09,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:09,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/76c31a2b1d3c4c32a41d6419f4fde9a0 2024-11-19T00:14:09,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/e1a5ff7ea72546aaaff4ec3199b49290 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/e1a5ff7ea72546aaaff4ec3199b49290 2024-11-19T00:14:09,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/e1a5ff7ea72546aaaff4ec3199b49290, entries=200, sequenceid=523, filesize=39.0 K 2024-11-19T00:14:09,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/be71b6655bcf46e8ae96860a99c40c6a as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/be71b6655bcf46e8ae96860a99c40c6a 2024-11-19T00:14:09,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/be71b6655bcf46e8ae96860a99c40c6a, entries=150, sequenceid=523, filesize=12.0 K 2024-11-19T00:14:09,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/76c31a2b1d3c4c32a41d6419f4fde9a0 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/76c31a2b1d3c4c32a41d6419f4fde9a0 2024-11-19T00:14:09,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/76c31a2b1d3c4c32a41d6419f4fde9a0, entries=150, sequenceid=523, filesize=12.0 K 2024-11-19T00:14:09,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 0f7b113264f1c67c501d9bd1480cc2b7 in 878ms, sequenceid=523, compaction requested=true 2024-11-19T00:14:09,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:09,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:14:09,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:09,306 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:09,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:14:09,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:09,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:14:09,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:09,306 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:09,307 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111869 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:09,307 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/A is initiating minor compaction (all files) 2024-11-19T00:14:09,307 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/A in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:09,307 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:09,307 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/B is initiating minor compaction (all files) 2024-11-19T00:14:09,308 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/44569da080414525900941f8dd96a33b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/e5ddf727c55a4c94a5c6b2d14525c281, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/e1a5ff7ea72546aaaff4ec3199b49290] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=109.2 K 2024-11-19T00:14:09,308 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/B in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:09,308 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:09,308 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/e0e89d7148ef4fada961dacb2a468322, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/e575a11a05c24d96abeff238ea096a1e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/be71b6655bcf46e8ae96860a99c40c6a] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=36.7 K 2024-11-19T00:14:09,308 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. files: [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/44569da080414525900941f8dd96a33b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/e5ddf727c55a4c94a5c6b2d14525c281, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/e1a5ff7ea72546aaaff4ec3199b49290] 2024-11-19T00:14:09,309 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting e0e89d7148ef4fada961dacb2a468322, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731975245780 2024-11-19T00:14:09,309 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44569da080414525900941f8dd96a33b, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731975245780 2024-11-19T00:14:09,309 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting e575a11a05c24d96abeff238ea096a1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731975246468 2024-11-19T00:14:09,309 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5ddf727c55a4c94a5c6b2d14525c281, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731975246461 2024-11-19T00:14:09,309 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1a5ff7ea72546aaaff4ec3199b49290, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731975247226 2024-11-19T00:14:09,309 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting be71b6655bcf46e8ae96860a99c40c6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731975247226 2024-11-19T00:14:09,318 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:09,330 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119a6422d1529f44167b70d3598b8d4cf54_0f7b113264f1c67c501d9bd1480cc2b7 store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:09,332 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119a6422d1529f44167b70d3598b8d4cf54_0f7b113264f1c67c501d9bd1480cc2b7, store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:09,332 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a6422d1529f44167b70d3598b8d4cf54_0f7b113264f1c67c501d9bd1480cc2b7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:09,336 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#B#compaction#178 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:09,336 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/48ac942838ac486b879b44ca5ee94c16 is 50, key is test_row_0/B:col10/1731975247226/Put/seqid=0 2024-11-19T00:14:09,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742041_1217 (size=4469) 2024-11-19T00:14:09,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742042_1218 (size=13119) 2024-11-19T00:14:09,357 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/48ac942838ac486b879b44ca5ee94c16 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/48ac942838ac486b879b44ca5ee94c16 2024-11-19T00:14:09,363 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/B of 0f7b113264f1c67c501d9bd1480cc2b7 into 48ac942838ac486b879b44ca5ee94c16(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:09,363 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:09,363 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/B, priority=13, startTime=1731975249306; duration=0sec 2024-11-19T00:14:09,364 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:09,364 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:B 2024-11-19T00:14:09,364 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:09,365 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:09,365 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/C is initiating minor compaction (all files) 2024-11-19T00:14:09,365 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/C in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:09,365 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/8107ea3e066242aa8ea17b438b062d56, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/b51b17b0096844ff8ae1ad5975f551c3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/76c31a2b1d3c4c32a41d6419f4fde9a0] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=36.7 K 2024-11-19T00:14:09,366 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 8107ea3e066242aa8ea17b438b062d56, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731975245780 2024-11-19T00:14:09,366 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting b51b17b0096844ff8ae1ad5975f551c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731975246468 2024-11-19T00:14:09,366 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 76c31a2b1d3c4c32a41d6419f4fde9a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731975247226 2024-11-19T00:14:09,384 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#C#compaction#179 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:09,385 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/c960ecfdc7bd41c99f1099d3ee17ca35 is 50, key is test_row_0/C:col10/1731975247226/Put/seqid=0 2024-11-19T00:14:09,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742043_1219 (size=13119) 2024-11-19T00:14:09,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T00:14:09,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:09,411 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 0f7b113264f1c67c501d9bd1480cc2b7 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-19T00:14:09,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=A 2024-11-19T00:14:09,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:09,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=B 2024-11-19T00:14:09,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:09,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=C 2024-11-19T00:14:09,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:09,420 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/c960ecfdc7bd41c99f1099d3ee17ca35 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/c960ecfdc7bd41c99f1099d3ee17ca35 2024-11-19T00:14:09,429 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/C of 0f7b113264f1c67c501d9bd1480cc2b7 into c960ecfdc7bd41c99f1099d3ee17ca35(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:09,429 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:09,429 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/C, priority=13, startTime=1731975249306; duration=0sec 2024-11-19T00:14:09,429 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:09,429 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:C 2024-11-19T00:14:09,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119423b519a5c74465caae01aeddbbbd055_0f7b113264f1c67c501d9bd1480cc2b7 is 50, key is test_row_0/A:col10/1731975248439/Put/seqid=0 2024-11-19T00:14:09,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742044_1220 (size=12454) 2024-11-19T00:14:09,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:09,484 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119423b519a5c74465caae01aeddbbbd055_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119423b519a5c74465caae01aeddbbbd055_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:09,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/3ada52b6f5ff4fad951707d0ca92ab2f, store: [table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:09,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/3ada52b6f5ff4fad951707d0ca92ab2f is 175, key is test_row_0/A:col10/1731975248439/Put/seqid=0 2024-11-19T00:14:09,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742045_1221 (size=31255) 2024-11-19T00:14:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:09,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. as already flushing 2024-11-19T00:14:09,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975249643, max timestamp=1731975249660Num uniques 859; 2024-11-19T00:14:09,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975249643, max timestamp=1731975249660Num uniques 859; 2024-11-19T00:14:09,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975249643, max timestamp=1731975249660Num uniques 859; 2024-11-19T00:14:09,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:14:09,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:14:09,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:14:09,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 383 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975309726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:09,748 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#A#compaction#177 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:09,748 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/075581c95bee4f53acf79881c85819f8 is 175, key is test_row_0/A:col10/1731975247226/Put/seqid=0 2024-11-19T00:14:09,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742046_1222 (size=32073) 2024-11-19T00:14:09,763 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/075581c95bee4f53acf79881c85819f8 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/075581c95bee4f53acf79881c85819f8 2024-11-19T00:14:09,767 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/A of 0f7b113264f1c67c501d9bd1480cc2b7 into 075581c95bee4f53acf79881c85819f8(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:09,767 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:09,767 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/A, priority=13, startTime=1731975249306; duration=0sec 2024-11-19T00:14:09,767 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:09,767 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:A 2024-11-19T00:14:09,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:09,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 384 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975309833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:09,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:09,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:09,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:09,929 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=533, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/3ada52b6f5ff4fad951707d0ca92ab2f 2024-11-19T00:14:09,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/59c869bce7b34119bce3d214f9cdd46b is 50, key is test_row_0/B:col10/1731975248439/Put/seqid=0 2024-11-19T00:14:09,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742047_1223 (size=12301) 2024-11-19T00:14:10,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:10,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 385 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975310044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:10,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:10,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:10,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T00:14:10,211 INFO [Thread-622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-622. 2024-11-19T00:14:10,211 DEBUG [Thread-622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:14:10,211 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:10,211 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:10,211 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:14:10,213 INFO [Thread-626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-626. 2024-11-19T00:14:10,214 DEBUG [Thread-626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:14:10,214 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:10,214 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:10,214 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:14:10,214 INFO [Thread-628 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-628. 2024-11-19T00:14:10,214 DEBUG [Thread-628 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:14:10,214 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:10,214 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:10,214 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:14:10,216 INFO [Thread-624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-624. 2024-11-19T00:14:10,216 DEBUG [Thread-624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:14:10,216 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:10,216 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:10,216 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:14:10,341 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/59c869bce7b34119bce3d214f9cdd46b 2024-11-19T00:14:10,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:10,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 386 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47022 deadline: 1731975310354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:10,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:10,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:10,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:10,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/e60732e2b8ce421f8009e21a624f44cb is 50, key is test_row_0/C:col10/1731975248439/Put/seqid=0 2024-11-19T00:14:10,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742048_1224 (size=12301) 2024-11-19T00:14:10,760 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/e60732e2b8ce421f8009e21a624f44cb 2024-11-19T00:14:10,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/3ada52b6f5ff4fad951707d0ca92ab2f as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/3ada52b6f5ff4fad951707d0ca92ab2f 2024-11-19T00:14:10,769 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/3ada52b6f5ff4fad951707d0ca92ab2f, entries=150, sequenceid=533, filesize=30.5 K 2024-11-19T00:14:10,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/59c869bce7b34119bce3d214f9cdd46b as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/59c869bce7b34119bce3d214f9cdd46b 2024-11-19T00:14:10,774 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/59c869bce7b34119bce3d214f9cdd46b, entries=150, sequenceid=533, filesize=12.0 K 2024-11-19T00:14:10,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/e60732e2b8ce421f8009e21a624f44cb as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/e60732e2b8ce421f8009e21a624f44cb 2024-11-19T00:14:10,779 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/e60732e2b8ce421f8009e21a624f44cb, entries=150, sequenceid=533, filesize=12.0 K 2024-11-19T00:14:10,780 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 0f7b113264f1c67c501d9bd1480cc2b7 in 1369ms, sequenceid=533, compaction requested=false 2024-11-19T00:14:10,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:10,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:10,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-19T00:14:10,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-19T00:14:10,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-19T00:14:10,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7590 sec 2024-11-19T00:14:10,789 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.7640 sec 2024-11-19T00:14:10,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:10,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f7b113264f1c67c501d9bd1480cc2b7 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-19T00:14:10,866 INFO [Thread-613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-613. 2024-11-19T00:14:10,866 DEBUG [Thread-613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:14:10,866 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:10,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=A 2024-11-19T00:14:10,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:14:10,866 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:10,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=B 2024-11-19T00:14:10,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:14:10,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=C 2024-11-19T00:14:10,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:14:10,867 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:14:10,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e0f2a77f26fd45c2b79049dd8be6d896_0f7b113264f1c67c501d9bd1480cc2b7 is 50, key is test_row_0/A:col10/1731975250864/Put/seqid=0 2024-11-19T00:14:10,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742049_1225 (size=12454) 2024-11-19T00:14:11,281 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:11,288 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e0f2a77f26fd45c2b79049dd8be6d896_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e0f2a77f26fd45c2b79049dd8be6d896_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:11,289 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/4d32f725060e47c68d636bbb231bf673, store: [table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:11,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/4d32f725060e47c68d636bbb231bf673 is 175, key is test_row_0/A:col10/1731975250864/Put/seqid=0 2024-11-19T00:14:11,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742050_1226 (size=31255) 2024-11-19T00:14:11,696 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=573, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/4d32f725060e47c68d636bbb231bf673 2024-11-19T00:14:11,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/fce9d3e8117745c2a5ce8296178e1b4f is 50, key is test_row_0/B:col10/1731975250864/Put/seqid=0 2024-11-19T00:14:11,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742051_1227 (size=12301) 2024-11-19T00:14:12,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=573 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/fce9d3e8117745c2a5ce8296178e1b4f 2024-11-19T00:14:12,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/b24c58ca698d43f5aacbcb89a781bdef is 50, key is test_row_0/C:col10/1731975250864/Put/seqid=0 2024-11-19T00:14:12,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742052_1228 (size=12301) 2024-11-19T00:14:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T00:14:12,204 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:14:12,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=573 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/b24c58ca698d43f5aacbcb89a781bdef 2024-11-19T00:14:12,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/4d32f725060e47c68d636bbb231bf673 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/4d32f725060e47c68d636bbb231bf673 2024-11-19T00:14:12,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/4d32f725060e47c68d636bbb231bf673, entries=150, sequenceid=573, filesize=30.5 K 2024-11-19T00:14:12,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/fce9d3e8117745c2a5ce8296178e1b4f as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/fce9d3e8117745c2a5ce8296178e1b4f 2024-11-19T00:14:12,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/fce9d3e8117745c2a5ce8296178e1b4f, entries=150, sequenceid=573, filesize=12.0 K 2024-11-19T00:14:12,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/b24c58ca698d43f5aacbcb89a781bdef as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/b24c58ca698d43f5aacbcb89a781bdef 2024-11-19T00:14:12,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/b24c58ca698d43f5aacbcb89a781bdef, entries=150, sequenceid=573, filesize=12.0 K 2024-11-19T00:14:12,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=0 B/0 for 0f7b113264f1c67c501d9bd1480cc2b7 in 1688ms, sequenceid=573, compaction requested=true 2024-11-19T00:14:12,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:12,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:14:12,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:12,554 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:12,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:14:12,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:12,554 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:12,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f7b113264f1c67c501d9bd1480cc2b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:14:12,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:12,560 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94583 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:12,560 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:12,560 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/B is initiating minor compaction (all files) 2024-11-19T00:14:12,560 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/A is initiating minor compaction (all files) 2024-11-19T00:14:12,560 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/B in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:12,560 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/A in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:12,561 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/48ac942838ac486b879b44ca5ee94c16, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/59c869bce7b34119bce3d214f9cdd46b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/fce9d3e8117745c2a5ce8296178e1b4f] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=36.8 K 2024-11-19T00:14:12,561 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/075581c95bee4f53acf79881c85819f8, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/3ada52b6f5ff4fad951707d0ca92ab2f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/4d32f725060e47c68d636bbb231bf673] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=92.4 K 2024-11-19T00:14:12,561 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:12,561 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. files: [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/075581c95bee4f53acf79881c85819f8, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/3ada52b6f5ff4fad951707d0ca92ab2f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/4d32f725060e47c68d636bbb231bf673] 2024-11-19T00:14:12,561 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 48ac942838ac486b879b44ca5ee94c16, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731975247226 2024-11-19T00:14:12,562 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 075581c95bee4f53acf79881c85819f8, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731975247226 2024-11-19T00:14:12,562 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 59c869bce7b34119bce3d214f9cdd46b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1731975248439 2024-11-19T00:14:12,562 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ada52b6f5ff4fad951707d0ca92ab2f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1731975248439 2024-11-19T00:14:12,562 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting fce9d3e8117745c2a5ce8296178e1b4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=573, earliestPutTs=1731975249713 2024-11-19T00:14:12,563 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d32f725060e47c68d636bbb231bf673, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=573, earliestPutTs=1731975249713 2024-11-19T00:14:12,573 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#B#compaction#186 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:12,574 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/ee449c78ba9a489ebcf226cfd89259b8 is 50, key is test_row_0/B:col10/1731975250864/Put/seqid=0 2024-11-19T00:14:12,583 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:12,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742053_1229 (size=13221) 2024-11-19T00:14:12,586 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411193215dca89511417f9e2c02bc96175c8e_0f7b113264f1c67c501d9bd1480cc2b7 store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:12,617 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411193215dca89511417f9e2c02bc96175c8e_0f7b113264f1c67c501d9bd1480cc2b7, store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:12,617 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193215dca89511417f9e2c02bc96175c8e_0f7b113264f1c67c501d9bd1480cc2b7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:12,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742054_1230 (size=4469) 2024-11-19T00:14:12,652 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T00:14:12,995 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/ee449c78ba9a489ebcf226cfd89259b8 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ee449c78ba9a489ebcf226cfd89259b8 2024-11-19T00:14:13,003 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/B of 0f7b113264f1c67c501d9bd1480cc2b7 into ee449c78ba9a489ebcf226cfd89259b8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:13,003 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:13,003 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/B, priority=13, startTime=1731975252554; duration=0sec 2024-11-19T00:14:13,004 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:13,004 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:B 2024-11-19T00:14:13,004 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:13,005 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:13,005 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 0f7b113264f1c67c501d9bd1480cc2b7/C is initiating minor compaction (all files) 2024-11-19T00:14:13,005 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f7b113264f1c67c501d9bd1480cc2b7/C in TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:13,005 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/c960ecfdc7bd41c99f1099d3ee17ca35, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/e60732e2b8ce421f8009e21a624f44cb, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/b24c58ca698d43f5aacbcb89a781bdef] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp, totalSize=36.8 K 2024-11-19T00:14:13,006 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting c960ecfdc7bd41c99f1099d3ee17ca35, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731975247226 2024-11-19T00:14:13,006 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting e60732e2b8ce421f8009e21a624f44cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1731975248439 2024-11-19T00:14:13,007 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting b24c58ca698d43f5aacbcb89a781bdef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=573, earliestPutTs=1731975249713 2024-11-19T00:14:13,019 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#C#compaction#188 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:13,019 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/f7bd3727c81f4d8f9edde3871508b299 is 50, key is test_row_0/C:col10/1731975250864/Put/seqid=0 2024-11-19T00:14:13,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742055_1231 (size=13221) 2024-11-19T00:14:13,023 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f7b113264f1c67c501d9bd1480cc2b7#A#compaction#187 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:13,023 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/50f2e1f9e0e445bfa8a7f011df690443 is 175, key is test_row_0/A:col10/1731975250864/Put/seqid=0 2024-11-19T00:14:13,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742056_1232 (size=32175) 2024-11-19T00:14:13,434 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/f7bd3727c81f4d8f9edde3871508b299 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/f7bd3727c81f4d8f9edde3871508b299 2024-11-19T00:14:13,437 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/50f2e1f9e0e445bfa8a7f011df690443 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/50f2e1f9e0e445bfa8a7f011df690443 2024-11-19T00:14:13,440 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/C of 0f7b113264f1c67c501d9bd1480cc2b7 into f7bd3727c81f4d8f9edde3871508b299(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:13,440 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:13,440 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/C, priority=13, startTime=1731975252555; duration=0sec 2024-11-19T00:14:13,440 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:13,441 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:C 2024-11-19T00:14:13,441 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f7b113264f1c67c501d9bd1480cc2b7/A of 0f7b113264f1c67c501d9bd1480cc2b7 into 50f2e1f9e0e445bfa8a7f011df690443(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:13,441 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f7b113264f1c67c501d9bd1480cc2b7: 2024-11-19T00:14:13,441 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., storeName=0f7b113264f1c67c501d9bd1480cc2b7/A, priority=13, startTime=1731975252554; duration=0sec 2024-11-19T00:14:13,442 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:13,442 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f7b113264f1c67c501d9bd1480cc2b7:A 2024-11-19T00:14:13,626 INFO [Thread-619 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-619. 2024-11-19T00:14:13,626 DEBUG [Thread-619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:14:13,626 DEBUG [Thread-619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:13,626 DEBUG [Thread-619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:13,626 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:14:13,665 INFO [Thread-615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-615. 2024-11-19T00:14:13,665 DEBUG [Thread-615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:14:13,665 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:13,665 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:13,665 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:14:13,716 INFO [Thread-617 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-617. 2024-11-19T00:14:13,716 DEBUG [Thread-617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:14:13,716 DEBUG [Thread-617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:13,716 DEBUG [Thread-617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:13,716 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:14:17,319 INFO [Thread-611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-611. 2024-11-19T00:14:17,319 DEBUG [Thread-611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:14:17,319 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:17,320 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:17,320 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:14:17,320 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T00:14:17,320 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-19T00:14:17,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 328 2024-11-19T00:14:17,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-19T00:14:17,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-19T00:14:17,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-19T00:14:17,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T00:14:17,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2819 2024-11-19T00:14:17,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2791 2024-11-19T00:14:17,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T00:14:17,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1183 2024-11-19T00:14:17,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3545 rows 2024-11-19T00:14:17,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1189 2024-11-19T00:14:17,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3565 rows 2024-11-19T00:14:17,322 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T00:14:17,322 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T00:14:17,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:17,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:17,323 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:14:17,324 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T00:14:17,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T00:14:17,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T00:14:17,329 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975257328"}]},"ts":"1731975257328"} 2024-11-19T00:14:17,330 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T00:14:17,330 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T00:14:17,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T00:14:17,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f7b113264f1c67c501d9bd1480cc2b7, UNASSIGN}] 2024-11-19T00:14:17,333 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f7b113264f1c67c501d9bd1480cc2b7, UNASSIGN 2024-11-19T00:14:17,333 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=0f7b113264f1c67c501d9bd1480cc2b7, regionState=CLOSING, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:14:17,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f7b113264f1c67c501d9bd1480cc2b7, UNASSIGN because future has completed 2024-11-19T00:14:17,335 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T00:14:17,335 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317}] 2024-11-19T00:14:17,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T00:14:17,488 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:17,488 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T00:14:17,489 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing 0f7b113264f1c67c501d9bd1480cc2b7, disabling compactions & flushes 2024-11-19T00:14:17,489 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:17,489 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:17,489 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. after waiting 0 ms 2024-11-19T00:14:17,489 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:17,489 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing 0f7b113264f1c67c501d9bd1480cc2b7 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-19T00:14:17,490 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=A 2024-11-19T00:14:17,490 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:17,490 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=B 2024-11-19T00:14:17,490 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:17,490 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f7b113264f1c67c501d9bd1480cc2b7, store=C 2024-11-19T00:14:17,491 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:17,501 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193f3012864a0047c1b63eeb5e8b65c556_0f7b113264f1c67c501d9bd1480cc2b7 is 50, key is test_row_0/A:col10/1731975257315/Put/seqid=0 2024-11-19T00:14:17,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742057_1233 (size=12454) 2024-11-19T00:14:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T00:14:17,907 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:17,918 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193f3012864a0047c1b63eeb5e8b65c556_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193f3012864a0047c1b63eeb5e8b65c556_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:17,920 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/ad14da54011d4487add33ceb580e392c, store: [table=TestAcidGuarantees family=A region=0f7b113264f1c67c501d9bd1480cc2b7] 2024-11-19T00:14:17,921 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/ad14da54011d4487add33ceb580e392c is 175, key is test_row_0/A:col10/1731975257315/Put/seqid=0 2024-11-19T00:14:17,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742058_1234 (size=31255) 2024-11-19T00:14:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T00:14:18,327 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=583, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/ad14da54011d4487add33ceb580e392c 2024-11-19T00:14:18,340 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/96d6e1b73dd14d3b81f023e20f48053a is 50, key is test_row_0/B:col10/1731975257315/Put/seqid=0 2024-11-19T00:14:18,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742059_1235 (size=12301) 2024-11-19T00:14:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T00:14:18,744 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=583 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/96d6e1b73dd14d3b81f023e20f48053a 2024-11-19T00:14:18,752 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/3cd7d33b98384bb594d36dcda012b3bf is 50, key is test_row_0/C:col10/1731975257315/Put/seqid=0 2024-11-19T00:14:18,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742060_1236 (size=12301) 2024-11-19T00:14:19,157 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=583 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/3cd7d33b98384bb594d36dcda012b3bf 2024-11-19T00:14:19,164 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/A/ad14da54011d4487add33ceb580e392c as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/ad14da54011d4487add33ceb580e392c 2024-11-19T00:14:19,169 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/ad14da54011d4487add33ceb580e392c, entries=150, sequenceid=583, filesize=30.5 K 2024-11-19T00:14:19,170 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/B/96d6e1b73dd14d3b81f023e20f48053a as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/96d6e1b73dd14d3b81f023e20f48053a 2024-11-19T00:14:19,175 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/96d6e1b73dd14d3b81f023e20f48053a, entries=150, sequenceid=583, filesize=12.0 K 2024-11-19T00:14:19,176 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/.tmp/C/3cd7d33b98384bb594d36dcda012b3bf as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/3cd7d33b98384bb594d36dcda012b3bf 2024-11-19T00:14:19,181 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/3cd7d33b98384bb594d36dcda012b3bf, entries=150, sequenceid=583, filesize=12.0 K 2024-11-19T00:14:19,182 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 0f7b113264f1c67c501d9bd1480cc2b7 in 1693ms, sequenceid=583, compaction requested=false 2024-11-19T00:14:19,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/0a1ee3b775c3400585e13dfd60e0ea16, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/feef0c23e2d844aaa475e3cc99f88065, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/92884ac7249d4cfe965968df850fba50, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/c4227c9be7cd4112bcea48394b357b0f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/ee0f30fb72734e2191c5b1919a8e6b2b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/632ca0d1991741288e4af821cdef87f7, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/52dca9e672c345d595731b81195fa486, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/0a30671bd74e4604bb64c449dfc6d0ab, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/2d991441ae51493bbec14b3a7815fd6c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/b7c7b39909894d638ec3bd697f6ecb32, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/779a9c63f4dc42afa0bf4766b5d50bda, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/1406fcbbf9eb45d1a5782f302f41ce23, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/4a3fab8788604c6694d1d24b58db3a86, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/a008098e03a44b1081956c8a6094e1a0, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/40eda8f1f31742348f3897ddebba00a5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/77f39d656d214d7f97355d3ae8913e90, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/e3f1c44626284d36a55bebf9a552950f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/fde9cda241b044d282e68bb587991c03, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/4527823ca32d4f88aa175ebc6a9ad175, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/23481213c39f498cb37051a92e6c2c6d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/43ca107fb81d4336a1a57478c00d5831, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/44569da080414525900941f8dd96a33b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/e5ddf727c55a4c94a5c6b2d14525c281, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/e1a5ff7ea72546aaaff4ec3199b49290, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/075581c95bee4f53acf79881c85819f8, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/3ada52b6f5ff4fad951707d0ca92ab2f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/4d32f725060e47c68d636bbb231bf673] to archive 2024-11-19T00:14:19,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T00:14:19,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/0a1ee3b775c3400585e13dfd60e0ea16 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/0a1ee3b775c3400585e13dfd60e0ea16 2024-11-19T00:14:19,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/feef0c23e2d844aaa475e3cc99f88065 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/feef0c23e2d844aaa475e3cc99f88065 2024-11-19T00:14:19,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/92884ac7249d4cfe965968df850fba50 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/92884ac7249d4cfe965968df850fba50 2024-11-19T00:14:19,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/c4227c9be7cd4112bcea48394b357b0f to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/c4227c9be7cd4112bcea48394b357b0f 2024-11-19T00:14:19,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/ee0f30fb72734e2191c5b1919a8e6b2b to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/ee0f30fb72734e2191c5b1919a8e6b2b 2024-11-19T00:14:19,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/632ca0d1991741288e4af821cdef87f7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/632ca0d1991741288e4af821cdef87f7 2024-11-19T00:14:19,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/52dca9e672c345d595731b81195fa486 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/52dca9e672c345d595731b81195fa486 2024-11-19T00:14:19,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/0a30671bd74e4604bb64c449dfc6d0ab to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/0a30671bd74e4604bb64c449dfc6d0ab 2024-11-19T00:14:19,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/2d991441ae51493bbec14b3a7815fd6c to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/2d991441ae51493bbec14b3a7815fd6c 2024-11-19T00:14:19,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/b7c7b39909894d638ec3bd697f6ecb32 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/b7c7b39909894d638ec3bd697f6ecb32 2024-11-19T00:14:19,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/779a9c63f4dc42afa0bf4766b5d50bda to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/779a9c63f4dc42afa0bf4766b5d50bda 2024-11-19T00:14:19,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/1406fcbbf9eb45d1a5782f302f41ce23 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/1406fcbbf9eb45d1a5782f302f41ce23 2024-11-19T00:14:19,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/4a3fab8788604c6694d1d24b58db3a86 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/4a3fab8788604c6694d1d24b58db3a86 2024-11-19T00:14:19,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/a008098e03a44b1081956c8a6094e1a0 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/a008098e03a44b1081956c8a6094e1a0 2024-11-19T00:14:19,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/40eda8f1f31742348f3897ddebba00a5 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/40eda8f1f31742348f3897ddebba00a5 2024-11-19T00:14:19,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/77f39d656d214d7f97355d3ae8913e90 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/77f39d656d214d7f97355d3ae8913e90 2024-11-19T00:14:19,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/e3f1c44626284d36a55bebf9a552950f to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/e3f1c44626284d36a55bebf9a552950f 2024-11-19T00:14:19,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/fde9cda241b044d282e68bb587991c03 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/fde9cda241b044d282e68bb587991c03 2024-11-19T00:14:19,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/4527823ca32d4f88aa175ebc6a9ad175 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/4527823ca32d4f88aa175ebc6a9ad175 2024-11-19T00:14:19,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/23481213c39f498cb37051a92e6c2c6d to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/23481213c39f498cb37051a92e6c2c6d 2024-11-19T00:14:19,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/43ca107fb81d4336a1a57478c00d5831 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/43ca107fb81d4336a1a57478c00d5831 2024-11-19T00:14:19,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/44569da080414525900941f8dd96a33b to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/44569da080414525900941f8dd96a33b 2024-11-19T00:14:19,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/e5ddf727c55a4c94a5c6b2d14525c281 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/e5ddf727c55a4c94a5c6b2d14525c281 2024-11-19T00:14:19,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/e1a5ff7ea72546aaaff4ec3199b49290 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/e1a5ff7ea72546aaaff4ec3199b49290 2024-11-19T00:14:19,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/075581c95bee4f53acf79881c85819f8 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/075581c95bee4f53acf79881c85819f8 2024-11-19T00:14:19,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/3ada52b6f5ff4fad951707d0ca92ab2f to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/3ada52b6f5ff4fad951707d0ca92ab2f 2024-11-19T00:14:19,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/4d32f725060e47c68d636bbb231bf673 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/4d32f725060e47c68d636bbb231bf673 2024-11-19T00:14:19,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/cc0531ad622d45f484d16d4b544598b8, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/b55d3cbb0d1e4574b9551c42ad1cf006, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/8d48d6551aad4e56a5e50683c985d6ad, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ffe1e0a694f34d31b92955d5bf28366b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/873d75a651b842039154d8b3669bed27, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/0cc530bd1d684baca741d0859faf821b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/7c8f70aaee1d45718c31cad1ac2a9288, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ddf85a6e26274765b51afdbf57c44172, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/79d8469c250141a6ae7be93af1115278, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/99ee6915b68c4b6689cc33ccdc2828f6, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/319dd584a3d64c6abe9d09f70d78543c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ad33f75fae884f768ce332a381a8a9f3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/d5f2ed0e7ec043c1bf4142f0dc0fdd1c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ea9be7b1e9f644428041b63b93a8dd56, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/7786291da16d4cf8886363d8b0506ad0, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/3eb66e34398847bfb07699115c5bc751, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/a05c083fac2a46c38837bf3e01bc2753, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/5c52ef5cc2414f11babd3e9740499441, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/2957713b73b14b58a2e766f21a23b984, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/d5ab497f8aec413c9c2c8746eea9f383, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/e0e89d7148ef4fada961dacb2a468322, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/756e26d9317d441eae4dbe3ffc9c8803, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/e575a11a05c24d96abeff238ea096a1e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/48ac942838ac486b879b44ca5ee94c16, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/be71b6655bcf46e8ae96860a99c40c6a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/59c869bce7b34119bce3d214f9cdd46b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/fce9d3e8117745c2a5ce8296178e1b4f] to archive 2024-11-19T00:14:19,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T00:14:19,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/cc0531ad622d45f484d16d4b544598b8 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/cc0531ad622d45f484d16d4b544598b8 2024-11-19T00:14:19,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/b55d3cbb0d1e4574b9551c42ad1cf006 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/b55d3cbb0d1e4574b9551c42ad1cf006 2024-11-19T00:14:19,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/8d48d6551aad4e56a5e50683c985d6ad to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/8d48d6551aad4e56a5e50683c985d6ad 2024-11-19T00:14:19,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ffe1e0a694f34d31b92955d5bf28366b to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ffe1e0a694f34d31b92955d5bf28366b 2024-11-19T00:14:19,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/873d75a651b842039154d8b3669bed27 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/873d75a651b842039154d8b3669bed27 2024-11-19T00:14:19,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/0cc530bd1d684baca741d0859faf821b to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/0cc530bd1d684baca741d0859faf821b 2024-11-19T00:14:19,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/7c8f70aaee1d45718c31cad1ac2a9288 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/7c8f70aaee1d45718c31cad1ac2a9288 2024-11-19T00:14:19,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ddf85a6e26274765b51afdbf57c44172 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ddf85a6e26274765b51afdbf57c44172 2024-11-19T00:14:19,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/79d8469c250141a6ae7be93af1115278 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/79d8469c250141a6ae7be93af1115278 2024-11-19T00:14:19,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/99ee6915b68c4b6689cc33ccdc2828f6 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/99ee6915b68c4b6689cc33ccdc2828f6 2024-11-19T00:14:19,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/319dd584a3d64c6abe9d09f70d78543c to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/319dd584a3d64c6abe9d09f70d78543c 2024-11-19T00:14:19,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ad33f75fae884f768ce332a381a8a9f3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ad33f75fae884f768ce332a381a8a9f3 2024-11-19T00:14:19,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/d5f2ed0e7ec043c1bf4142f0dc0fdd1c to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/d5f2ed0e7ec043c1bf4142f0dc0fdd1c 2024-11-19T00:14:19,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ea9be7b1e9f644428041b63b93a8dd56 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ea9be7b1e9f644428041b63b93a8dd56 2024-11-19T00:14:19,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/7786291da16d4cf8886363d8b0506ad0 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/7786291da16d4cf8886363d8b0506ad0 2024-11-19T00:14:19,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/3eb66e34398847bfb07699115c5bc751 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/3eb66e34398847bfb07699115c5bc751 2024-11-19T00:14:19,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/a05c083fac2a46c38837bf3e01bc2753 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/a05c083fac2a46c38837bf3e01bc2753 2024-11-19T00:14:19,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/5c52ef5cc2414f11babd3e9740499441 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/5c52ef5cc2414f11babd3e9740499441 2024-11-19T00:14:19,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/2957713b73b14b58a2e766f21a23b984 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/2957713b73b14b58a2e766f21a23b984 2024-11-19T00:14:19,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/d5ab497f8aec413c9c2c8746eea9f383 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/d5ab497f8aec413c9c2c8746eea9f383 2024-11-19T00:14:19,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/e0e89d7148ef4fada961dacb2a468322 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/e0e89d7148ef4fada961dacb2a468322 2024-11-19T00:14:19,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/756e26d9317d441eae4dbe3ffc9c8803 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/756e26d9317d441eae4dbe3ffc9c8803 2024-11-19T00:14:19,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/e575a11a05c24d96abeff238ea096a1e to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/e575a11a05c24d96abeff238ea096a1e 2024-11-19T00:14:19,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/48ac942838ac486b879b44ca5ee94c16 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/48ac942838ac486b879b44ca5ee94c16 2024-11-19T00:14:19,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/be71b6655bcf46e8ae96860a99c40c6a to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/be71b6655bcf46e8ae96860a99c40c6a 2024-11-19T00:14:19,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/59c869bce7b34119bce3d214f9cdd46b to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/59c869bce7b34119bce3d214f9cdd46b 2024-11-19T00:14:19,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/fce9d3e8117745c2a5ce8296178e1b4f to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/fce9d3e8117745c2a5ce8296178e1b4f 2024-11-19T00:14:19,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/c0591aa69ea8442ebbd46f924b78ac79, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/b0aad806c5ec47afaf987dde33af38e4, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/e5a79dcb857147c896ca983917e745de, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/a8c12167d2a547ecac9b2e38c8f3127e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/0e63459c39e44440a479838c99f62b8f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/31023d0047214ef19aa9f1ebd47a5204, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/4cc302c5e5f74eca95368852721a5c3f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/42c34a9398e7468c9833ce0c4efdb09e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/e219cb785e914fb6bdd12a486db80abb, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/6e6d74878553400bb9220a2b465fb404, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/789c8c73feaf49e593286de256f9d8ab, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/d5e0f6ccf5814edf926cdc2c28958c1b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/d2db5b407d224e17b56091ee4ac2df69, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/72f3d71fd1994c4e8cbd5feb3832844a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/1adc1db07fb34e479a2482988c286795, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/f1c3311e36d248278d97bc6bff1d5f5a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/386e63deb2e14dfaa92d24e91127b135, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/cfdb95a0317e4129bb7f363ddd70b359, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/1c3b0307178f4b9897368e737a0d5e02, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/e35fe3d84fbc4e9ebba48db4767957ec, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/8107ea3e066242aa8ea17b438b062d56, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/5e9f6033d53c4786a042261e6d09c34e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/b51b17b0096844ff8ae1ad5975f551c3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/c960ecfdc7bd41c99f1099d3ee17ca35, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/76c31a2b1d3c4c32a41d6419f4fde9a0, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/e60732e2b8ce421f8009e21a624f44cb, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/b24c58ca698d43f5aacbcb89a781bdef] to archive 2024-11-19T00:14:19,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T00:14:19,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/c0591aa69ea8442ebbd46f924b78ac79 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/c0591aa69ea8442ebbd46f924b78ac79 2024-11-19T00:14:19,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/b0aad806c5ec47afaf987dde33af38e4 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/b0aad806c5ec47afaf987dde33af38e4 2024-11-19T00:14:19,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/e5a79dcb857147c896ca983917e745de to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/e5a79dcb857147c896ca983917e745de 2024-11-19T00:14:19,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/a8c12167d2a547ecac9b2e38c8f3127e to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/a8c12167d2a547ecac9b2e38c8f3127e 2024-11-19T00:14:19,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/0e63459c39e44440a479838c99f62b8f to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/0e63459c39e44440a479838c99f62b8f 2024-11-19T00:14:19,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/31023d0047214ef19aa9f1ebd47a5204 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/31023d0047214ef19aa9f1ebd47a5204 2024-11-19T00:14:19,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/4cc302c5e5f74eca95368852721a5c3f to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/4cc302c5e5f74eca95368852721a5c3f 2024-11-19T00:14:19,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/42c34a9398e7468c9833ce0c4efdb09e to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/42c34a9398e7468c9833ce0c4efdb09e 2024-11-19T00:14:19,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/e219cb785e914fb6bdd12a486db80abb to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/e219cb785e914fb6bdd12a486db80abb 2024-11-19T00:14:19,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/6e6d74878553400bb9220a2b465fb404 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/6e6d74878553400bb9220a2b465fb404 2024-11-19T00:14:19,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/789c8c73feaf49e593286de256f9d8ab to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/789c8c73feaf49e593286de256f9d8ab 2024-11-19T00:14:19,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/d5e0f6ccf5814edf926cdc2c28958c1b to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/d5e0f6ccf5814edf926cdc2c28958c1b 2024-11-19T00:14:19,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/d2db5b407d224e17b56091ee4ac2df69 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/d2db5b407d224e17b56091ee4ac2df69 2024-11-19T00:14:19,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/72f3d71fd1994c4e8cbd5feb3832844a to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/72f3d71fd1994c4e8cbd5feb3832844a 2024-11-19T00:14:19,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/1adc1db07fb34e479a2482988c286795 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/1adc1db07fb34e479a2482988c286795 2024-11-19T00:14:19,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/f1c3311e36d248278d97bc6bff1d5f5a to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/f1c3311e36d248278d97bc6bff1d5f5a 2024-11-19T00:14:19,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/386e63deb2e14dfaa92d24e91127b135 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/386e63deb2e14dfaa92d24e91127b135 2024-11-19T00:14:19,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/cfdb95a0317e4129bb7f363ddd70b359 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/cfdb95a0317e4129bb7f363ddd70b359 2024-11-19T00:14:19,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/1c3b0307178f4b9897368e737a0d5e02 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/1c3b0307178f4b9897368e737a0d5e02 2024-11-19T00:14:19,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/e35fe3d84fbc4e9ebba48db4767957ec to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/e35fe3d84fbc4e9ebba48db4767957ec 2024-11-19T00:14:19,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/8107ea3e066242aa8ea17b438b062d56 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/8107ea3e066242aa8ea17b438b062d56 2024-11-19T00:14:19,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/5e9f6033d53c4786a042261e6d09c34e to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/5e9f6033d53c4786a042261e6d09c34e 2024-11-19T00:14:19,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/b51b17b0096844ff8ae1ad5975f551c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/b51b17b0096844ff8ae1ad5975f551c3 2024-11-19T00:14:19,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/c960ecfdc7bd41c99f1099d3ee17ca35 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/c960ecfdc7bd41c99f1099d3ee17ca35 2024-11-19T00:14:19,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/76c31a2b1d3c4c32a41d6419f4fde9a0 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/76c31a2b1d3c4c32a41d6419f4fde9a0 2024-11-19T00:14:19,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/e60732e2b8ce421f8009e21a624f44cb to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/e60732e2b8ce421f8009e21a624f44cb 2024-11-19T00:14:19,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/b24c58ca698d43f5aacbcb89a781bdef to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/b24c58ca698d43f5aacbcb89a781bdef 2024-11-19T00:14:19,266 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/recovered.edits/586.seqid, newMaxSeqId=586, maxSeqId=4 2024-11-19T00:14:19,267 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. 2024-11-19T00:14:19,267 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for 0f7b113264f1c67c501d9bd1480cc2b7: Waiting for close lock at 1731975257489Running coprocessor pre-close hooks at 1731975257489Disabling compacts and flushes for region at 1731975257489Disabling writes for close at 1731975257489Obtaining lock to block concurrent updates at 1731975257489Preparing flush snapshotting stores in 0f7b113264f1c67c501d9bd1480cc2b7 at 1731975257489Finished memstore snapshotting TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731975257491 (+2 ms)Flushing stores of TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7. at 1731975257492 (+1 ms)Flushing 0f7b113264f1c67c501d9bd1480cc2b7/A: creating writer at 1731975257492Flushing mob file 0f7b113264f1c67c501d9bd1480cc2b7/A: appending metadata at 1731975257501 (+9 ms)Flushing mob file 0f7b113264f1c67c501d9bd1480cc2b7/A: closing flushed file at 1731975257501Flushing 0f7b113264f1c67c501d9bd1480cc2b7/A: appending metadata at 1731975257920 (+419 ms)Flushing 0f7b113264f1c67c501d9bd1480cc2b7/A: closing flushed file at 1731975257920Flushing 0f7b113264f1c67c501d9bd1480cc2b7/B: creating writer at 1731975258337 (+417 ms)Flushing 0f7b113264f1c67c501d9bd1480cc2b7/B: appending metadata at 1731975258340 (+3 ms)Flushing 0f7b113264f1c67c501d9bd1480cc2b7/B: closing flushed file at 1731975258340Flushing 0f7b113264f1c67c501d9bd1480cc2b7/C: creating writer at 1731975258749 (+409 ms)Flushing 0f7b113264f1c67c501d9bd1480cc2b7/C: appending metadata at 1731975258751 (+2 ms)Flushing 0f7b113264f1c67c501d9bd1480cc2b7/C: closing flushed file at 1731975258751Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7452ad78: reopening flushed file at 1731975259163 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5662945a: reopening flushed file at 1731975259170 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@151cbbda: reopening flushed file at 1731975259176 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 0f7b113264f1c67c501d9bd1480cc2b7 in 1693ms, sequenceid=583, compaction requested=false at 1731975259182 (+6 ms)Writing region close event to WAL at 1731975259264 (+82 ms)Running coprocessor post-close hooks at 1731975259267 (+3 ms)Closed at 1731975259267 2024-11-19T00:14:19,269 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed 0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:19,269 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=0f7b113264f1c67c501d9bd1480cc2b7, regionState=CLOSED 2024-11-19T00:14:19,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 because future has completed 2024-11-19T00:14:19,272 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-19T00:14:19,272 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure 0f7b113264f1c67c501d9bd1480cc2b7, server=6b1531871d35,38567,1731975194317 in 1.9360 sec 2024-11-19T00:14:19,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-19T00:14:19,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f7b113264f1c67c501d9bd1480cc2b7, UNASSIGN in 1.9400 sec 2024-11-19T00:14:19,276 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-19T00:14:19,276 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9430 sec 2024-11-19T00:14:19,277 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975259277"}]},"ts":"1731975259277"} 2024-11-19T00:14:19,279 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T00:14:19,279 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T00:14:19,281 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9550 sec 2024-11-19T00:14:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T00:14:19,474 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:14:19,474 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T00:14:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:14:19,477 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:14:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T00:14:19,478 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:14:19,481 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:19,486 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A, FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B, FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C, FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/recovered.edits] 2024-11-19T00:14:19,491 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/50f2e1f9e0e445bfa8a7f011df690443 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/50f2e1f9e0e445bfa8a7f011df690443 2024-11-19T00:14:19,493 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/ad14da54011d4487add33ceb580e392c to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/A/ad14da54011d4487add33ceb580e392c 2024-11-19T00:14:19,498 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/96d6e1b73dd14d3b81f023e20f48053a to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/96d6e1b73dd14d3b81f023e20f48053a 2024-11-19T00:14:19,499 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ee449c78ba9a489ebcf226cfd89259b8 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/B/ee449c78ba9a489ebcf226cfd89259b8 2024-11-19T00:14:19,502 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/3cd7d33b98384bb594d36dcda012b3bf to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/3cd7d33b98384bb594d36dcda012b3bf 2024-11-19T00:14:19,503 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/f7bd3727c81f4d8f9edde3871508b299 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/C/f7bd3727c81f4d8f9edde3871508b299 2024-11-19T00:14:19,505 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/recovered.edits/586.seqid to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7/recovered.edits/586.seqid 2024-11-19T00:14:19,505 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:19,506 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T00:14:19,506 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T00:14:19,507 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T00:14:19,511 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190000385f26f647eaa6fd726118f50ea3_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190000385f26f647eaa6fd726118f50ea3_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:19,512 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111915761b68da49433f927add99462eae6b_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111915761b68da49433f927add99462eae6b_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:19,513 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192f64c7773d54474fa29c13f844dfb592_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192f64c7773d54474fa29c13f844dfb592_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:19,514 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192fb0938e06784cf9a023d1521e339580_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192fb0938e06784cf9a023d1521e339580_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:19,515 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119365c72462fe648ce95e94d1acc3a3079_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119365c72462fe648ce95e94d1acc3a3079_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:19,516 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119388d82665d8b4e17aa4640130646148d_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119388d82665d8b4e17aa4640130646148d_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:19,517 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193e050108a25c42cfbdb6f561c1067d34_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193e050108a25c42cfbdb6f561c1067d34_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:19,518 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193f3012864a0047c1b63eeb5e8b65c556_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193f3012864a0047c1b63eeb5e8b65c556_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:19,519 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119423b519a5c74465caae01aeddbbbd055_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119423b519a5c74465caae01aeddbbbd055_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:19,519 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119522ee531a7a44e1e98d745ac8d54eecc_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119522ee531a7a44e1e98d745ac8d54eecc_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:19,520 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119606e3399c9a345b3b6ec55ee754f2d24_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119606e3399c9a345b3b6ec55ee754f2d24_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:19,522 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111960d1331504924baebe628f28e72b5375_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111960d1331504924baebe628f28e72b5375_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:19,523 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119643955bc91e542c7ad61e87a6155de28_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119643955bc91e542c7ad61e87a6155de28_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:19,524 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119985d61759d014cce805e72c87215a1a6_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119985d61759d014cce805e72c87215a1a6_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:19,525 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a0b859d6fc1a4db1953129f32ae8ece7_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a0b859d6fc1a4db1953129f32ae8ece7_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:19,527 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a8d283b30fbb47599cdf9bbffc8be1f7_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a8d283b30fbb47599cdf9bbffc8be1f7_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:19,528 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b741d92d752b461c986dcb36680f27a0_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b741d92d752b461c986dcb36680f27a0_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:19,529 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e0f2a77f26fd45c2b79049dd8be6d896_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e0f2a77f26fd45c2b79049dd8be6d896_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:19,530 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e8fe934c4ebd4596b402606f99eb00a6_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e8fe934c4ebd4596b402606f99eb00a6_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:19,531 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fd3adb4176dc4a2b89514ee2b2336955_0f7b113264f1c67c501d9bd1480cc2b7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fd3adb4176dc4a2b89514ee2b2336955_0f7b113264f1c67c501d9bd1480cc2b7 2024-11-19T00:14:19,531 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T00:14:19,533 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:14:19,536 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T00:14:19,538 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T00:14:19,539 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:14:19,539 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T00:14:19,539 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731975259539"}]},"ts":"9223372036854775807"} 2024-11-19T00:14:19,541 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T00:14:19,541 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0f7b113264f1c67c501d9bd1480cc2b7, NAME => 'TestAcidGuarantees,,1731975227002.0f7b113264f1c67c501d9bd1480cc2b7.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T00:14:19,541 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T00:14:19,541 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731975259541"}]},"ts":"9223372036854775807"} 2024-11-19T00:14:19,543 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T00:14:19,544 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:14:19,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 70 msec 2024-11-19T00:14:19,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T00:14:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T00:14:19,584 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:14:19,594 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 247) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_625823604_22 at /127.0.0.1:38306 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/cluster_ec2aa8be-d8f2-2fc7-fbd3-add83417a4c1/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/cluster_ec2aa8be-d8f2-2fc7-fbd3-add83417a4c1/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=460 (was 367) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2461 (was 2657) 2024-11-19T00:14:19,604 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=460, ProcessCount=11, AvailableMemoryMB=2461 2024-11-19T00:14:19,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T00:14:19,606 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T00:14:19,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T00:14:19,609 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T00:14:19,609 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:19,609 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-19T00:14:19,610 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T00:14:19,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-19T00:14:19,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742061_1237 (size=1061) 2024-11-19T00:14:19,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-19T00:14:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-19T00:14:20,022 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2 2024-11-19T00:14:20,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742062_1238 (size=53) 2024-11-19T00:14:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-19T00:14:20,434 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:14:20,435 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c24634f4d3f0bb64baf797330c06cc91, disabling compactions & flushes 2024-11-19T00:14:20,435 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:20,435 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:20,435 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. after waiting 0 ms 2024-11-19T00:14:20,435 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:20,435 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:20,435 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c24634f4d3f0bb64baf797330c06cc91: Waiting for close lock at 1731975260435Disabling compacts and flushes for region at 1731975260435Disabling writes for close at 1731975260435Writing region close event to WAL at 1731975260435Closed at 1731975260435 2024-11-19T00:14:20,439 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T00:14:20,439 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731975260439"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731975260439"}]},"ts":"1731975260439"} 2024-11-19T00:14:20,442 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T00:14:20,443 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T00:14:20,443 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975260443"}]},"ts":"1731975260443"} 2024-11-19T00:14:20,445 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T00:14:20,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c24634f4d3f0bb64baf797330c06cc91, ASSIGN}] 2024-11-19T00:14:20,447 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c24634f4d3f0bb64baf797330c06cc91, ASSIGN 2024-11-19T00:14:20,447 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c24634f4d3f0bb64baf797330c06cc91, ASSIGN; state=OFFLINE, location=6b1531871d35,38567,1731975194317; forceNewPlan=false, retain=false 2024-11-19T00:14:20,599 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=c24634f4d3f0bb64baf797330c06cc91, regionState=OPENING, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:14:20,603 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c24634f4d3f0bb64baf797330c06cc91, ASSIGN because future has completed 2024-11-19T00:14:20,603 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317}] 2024-11-19T00:14:20,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-19T00:14:20,761 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:20,762 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} 2024-11-19T00:14:20,762 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:20,762 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:14:20,762 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:20,762 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:20,763 INFO [StoreOpener-c24634f4d3f0bb64baf797330c06cc91-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:20,765 INFO [StoreOpener-c24634f4d3f0bb64baf797330c06cc91-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:14:20,765 INFO [StoreOpener-c24634f4d3f0bb64baf797330c06cc91-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c24634f4d3f0bb64baf797330c06cc91 columnFamilyName A 2024-11-19T00:14:20,765 DEBUG [StoreOpener-c24634f4d3f0bb64baf797330c06cc91-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:20,765 INFO [StoreOpener-c24634f4d3f0bb64baf797330c06cc91-1 {}] regionserver.HStore(327): Store=c24634f4d3f0bb64baf797330c06cc91/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:14:20,765 INFO [StoreOpener-c24634f4d3f0bb64baf797330c06cc91-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:20,767 INFO [StoreOpener-c24634f4d3f0bb64baf797330c06cc91-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:14:20,767 INFO [StoreOpener-c24634f4d3f0bb64baf797330c06cc91-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c24634f4d3f0bb64baf797330c06cc91 columnFamilyName B 2024-11-19T00:14:20,767 DEBUG [StoreOpener-c24634f4d3f0bb64baf797330c06cc91-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:20,767 INFO [StoreOpener-c24634f4d3f0bb64baf797330c06cc91-1 {}] regionserver.HStore(327): Store=c24634f4d3f0bb64baf797330c06cc91/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:14:20,767 INFO [StoreOpener-c24634f4d3f0bb64baf797330c06cc91-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:20,768 INFO [StoreOpener-c24634f4d3f0bb64baf797330c06cc91-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:14:20,768 INFO [StoreOpener-c24634f4d3f0bb64baf797330c06cc91-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c24634f4d3f0bb64baf797330c06cc91 columnFamilyName C 2024-11-19T00:14:20,768 DEBUG [StoreOpener-c24634f4d3f0bb64baf797330c06cc91-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:20,769 INFO [StoreOpener-c24634f4d3f0bb64baf797330c06cc91-1 {}] regionserver.HStore(327): Store=c24634f4d3f0bb64baf797330c06cc91/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:14:20,769 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:20,769 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:20,769 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:20,770 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:20,771 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:20,771 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:20,771 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T00:14:20,772 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:20,778 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T00:14:20,779 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened c24634f4d3f0bb64baf797330c06cc91; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73878398, jitterRate=0.1008739173412323}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T00:14:20,779 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:20,779 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for c24634f4d3f0bb64baf797330c06cc91: Running coprocessor pre-open hook at 1731975260762Writing region info on filesystem at 1731975260762Initializing all the Stores at 1731975260763 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975260763Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975260763Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975260763Cleaning up temporary data from old regions at 1731975260771 (+8 ms)Running coprocessor post-open hooks at 1731975260779 (+8 ms)Region opened successfully at 1731975260779 2024-11-19T00:14:20,780 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., pid=63, masterSystemTime=1731975260758 2024-11-19T00:14:20,782 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:20,782 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:20,782 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=c24634f4d3f0bb64baf797330c06cc91, regionState=OPEN, openSeqNum=2, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:14:20,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 because future has completed 2024-11-19T00:14:20,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-19T00:14:20,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 in 181 msec 2024-11-19T00:14:20,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-19T00:14:20,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c24634f4d3f0bb64baf797330c06cc91, ASSIGN in 341 msec 2024-11-19T00:14:20,789 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T00:14:20,789 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975260789"}]},"ts":"1731975260789"} 2024-11-19T00:14:20,791 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T00:14:20,791 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T00:14:20,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1850 sec 2024-11-19T00:14:21,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-19T00:14:21,764 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:14:21,765 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,765 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:14:21,765 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:14:21,766 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:14:21,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:14:21,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:14:21,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35a0de50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:14:21,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:14:21,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:21,768 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:14:21,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@326829c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,769 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:14:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:14:21,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:14:21,771 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:14:21,772 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T00:14:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 6b1531871d35,33843,1731975193570 2024-11-19T00:14:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@23d688bb 2024-11-19T00:14:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T00:14:21,774 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T00:14:21,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,776 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:14:21,776 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:14:21,777 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:14:21,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:14:21,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:14:21,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c58243c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:14:21,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:14:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:21,778 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:14:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46af1987, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,779 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:14:21,780 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:14:21,780 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:14:21,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:14:21,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:14:21,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@161ea75d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:14:21,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:14:21,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:21,782 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:14:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51c302ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,783 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:14:21,784 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:14:21,784 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:14:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:14:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:14:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64bc6f26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:14:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:14:21,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:21,786 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:14:21,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30d5723d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,788 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:14:21,788 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:14:21,789 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:14:21,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:14:21,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:14:21,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9a22946, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:14:21,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:14:21,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:21,790 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:14:21,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@da5e34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,792 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,792 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:14:21,792 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:14:21,793 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:14:21,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:14:21,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:14:21,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a3a935c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:14:21,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:14:21,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:21,794 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:14:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d0856f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,796 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:14:21,796 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:14:21,797 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:14:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:14:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:14:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51212327, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:14:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:14:21,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:21,798 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:14:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ea06700, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,800 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:14:21,800 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:14:21,801 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:14:21,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:14:21,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:14:21,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@93809fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:14:21,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:14:21,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:21,803 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:14:21,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41bf54e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,805 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:14:21,805 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:14:21,806 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:14:21,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:14:21,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:14:21,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f214bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:14:21,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:14:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:21,809 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:14:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10953dff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,811 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,811 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:14:21,811 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:14:21,812 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:14:21,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:14:21,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:14:21,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fc7ed8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:14:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:14:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:21,814 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:14:21,815 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7071a75f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,815 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:14:21,816 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:14:21,816 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:14:21,817 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:14:21,817 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:14:21,817 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c2961ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,817 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:14:21,817 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:14:21,817 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:21,818 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:14:21,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b2063d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:21,819 DEBUG [Thread-1115 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:14:21,820 DEBUG [Thread-1119 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:14:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:14:21,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:14:21,821 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:14:21,822 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:14:21,822 DEBUG [Thread-1124 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:14:21,823 DEBUG [Thread-1128 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:14:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-19T00:14:21,824 DEBUG [Thread-1119 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:14:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:14:21,824 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:14:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:14:21,824 DEBUG [Thread-1119 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:14:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:14:21,825 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:14:21,825 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:14:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T00:14:21,826 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:14:21,826 DEBUG [Thread-1122 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:14:21,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:14:21,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:14:21,826 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:14:21,826 DEBUG [Thread-1111 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:14:21,827 DEBUG [Thread-1122 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:14:21,827 DEBUG [Thread-1122 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:14:21,827 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:14:21,827 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:14:21,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:14:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:14:21,828 DEBUG [Thread-1130 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:14:21,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:14:21,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:14:21,829 DEBUG [Thread-1113 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:14:21,829 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:14:21,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:14:21,829 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:14:21,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:14:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:14:21,831 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:14:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:14:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:14:21,832 DEBUG [Thread-1117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:14:21,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:14:21,833 DEBUG [Thread-1126 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:14:21,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:14:21,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:14:21,834 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:14:21,834 DEBUG [Thread-1126 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:14:21,834 DEBUG [Thread-1117 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:14:21,835 DEBUG [Thread-1126 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:14:21,836 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:14:21,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:14:21,836 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:14:21,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:14:21,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:14:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:21,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c24634f4d3f0bb64baf797330c06cc91 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T00:14:21,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=A 2024-11-19T00:14:21,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:21,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=B 2024-11-19T00:14:21,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:21,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=C 2024-11-19T00:14:21,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:21,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/ecce0326ec564a7e95be9eb28e5e50a5 is 50, key is test_row_0/A:col10/1731975261839/Put/seqid=0 2024-11-19T00:14:21,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:21,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975321870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:21,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:21,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:21,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:21,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:21,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975321872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:21,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:21,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:21,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731975261868, max timestamp=1731975261876Num uniques 859; 2024-11-19T00:14:21,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:21,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731975261868, max timestamp=1731975261876Num uniques 859; 2024-11-19T00:14:21,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:14:21,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:14:21,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731975261868, max timestamp=1731975261876Num uniques 859; 2024-11-19T00:14:21,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:14:21,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742063_1239 (size=12001) 2024-11-19T00:14:21,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/ecce0326ec564a7e95be9eb28e5e50a5 2024-11-19T00:14:21,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/1186e2e1d1f446a0874387e6eb317576 is 50, key is test_row_0/B:col10/1731975261839/Put/seqid=0 2024-11-19T00:14:21,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51968 deadline: 1731975321923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:21,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1731975321923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:21,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51992 deadline: 1731975321923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:21,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742064_1240 (size=12001) 2024-11-19T00:14:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T00:14:21,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/1186e2e1d1f446a0874387e6eb317576 2024-11-19T00:14:21,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/37dcd6b9f793442184c6e5a4791f4477 is 50, key is test_row_0/C:col10/1731975261839/Put/seqid=0 2024-11-19T00:14:21,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T00:14:21,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:21,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:21,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:21,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:21,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:21,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975321984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:21,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:21,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:21,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:21,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975321985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:22,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742065_1241 (size=12001) 2024-11-19T00:14:22,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1731975322034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:22,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51992 deadline: 1731975322034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:22,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51968 deadline: 1731975322034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:22,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T00:14:22,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:22,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:22,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:22,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:22,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T00:14:22,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975322193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975322194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:22,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:22,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1731975322244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51992 deadline: 1731975322244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:22,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51968 deadline: 1731975322245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:22,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:22,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T00:14:22,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:22,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:22,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:22,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:22,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:22,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/37dcd6b9f793442184c6e5a4791f4477 2024-11-19T00:14:22,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/ecce0326ec564a7e95be9eb28e5e50a5 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/ecce0326ec564a7e95be9eb28e5e50a5 2024-11-19T00:14:22,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/ecce0326ec564a7e95be9eb28e5e50a5, entries=150, sequenceid=13, filesize=11.7 K 2024-11-19T00:14:22,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/1186e2e1d1f446a0874387e6eb317576 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/1186e2e1d1f446a0874387e6eb317576 2024-11-19T00:14:22,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T00:14:22,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:22,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:22,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:22,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:22,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T00:14:22,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/1186e2e1d1f446a0874387e6eb317576, entries=150, sequenceid=13, filesize=11.7 K 2024-11-19T00:14:22,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/37dcd6b9f793442184c6e5a4791f4477 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/37dcd6b9f793442184c6e5a4791f4477 2024-11-19T00:14:22,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/37dcd6b9f793442184c6e5a4791f4477, entries=150, sequenceid=13, filesize=11.7 K 2024-11-19T00:14:22,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for c24634f4d3f0bb64baf797330c06cc91 in 633ms, sequenceid=13, compaction requested=false 2024-11-19T00:14:22,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:22,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c24634f4d3f0bb64baf797330c06cc91 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-19T00:14:22,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=A 2024-11-19T00:14:22,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:14:22,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=B 2024-11-19T00:14:22,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:14:22,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=C 2024-11-19T00:14:22,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:14:22,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/23fcc7b6131a4af69dc5284cb5655190 is 50, key is test_row_0/A:col10/1731975261912/Put/seqid=0 2024-11-19T00:14:22,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975322542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:22,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975322543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:22,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51992 deadline: 1731975322554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:22,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:22,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51968 deadline: 1731975322554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:22,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:22,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1731975322554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:22,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:22,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742066_1242 (size=12001) 2024-11-19T00:14:22,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T00:14:22,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:22,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:22,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:22,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:22,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:22,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975322653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:22,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975322653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:22,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T00:14:22,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:22,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:22,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:22,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:22,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:22,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975322863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:22,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:22,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975322864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:22,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:22,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:22,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T00:14:22,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:22,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:22,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:22,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:22,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T00:14:22,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/23fcc7b6131a4af69dc5284cb5655190 2024-11-19T00:14:22,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/264f3b1854404e1a8de0ab7ff1117d51 is 50, key is test_row_0/B:col10/1731975261912/Put/seqid=0 2024-11-19T00:14:23,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742067_1243 (size=12001) 2024-11-19T00:14:23,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/264f3b1854404e1a8de0ab7ff1117d51 2024-11-19T00:14:23,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/3eb2cc4269514444b3520a23ea2974c3 is 50, key is test_row_0/C:col10/1731975261912/Put/seqid=0 2024-11-19T00:14:23,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T00:14:23,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:23,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:23,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:23,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:23,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:23,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51968 deadline: 1731975323064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:23,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:23,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:23,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:23,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51992 deadline: 1731975323064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:23,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:23,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:23,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:23,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1731975323074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:23,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742068_1244 (size=12001) 2024-11-19T00:14:23,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/3eb2cc4269514444b3520a23ea2974c3 2024-11-19T00:14:23,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/23fcc7b6131a4af69dc5284cb5655190 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/23fcc7b6131a4af69dc5284cb5655190 2024-11-19T00:14:23,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/23fcc7b6131a4af69dc5284cb5655190, entries=150, sequenceid=49, filesize=11.7 K 2024-11-19T00:14:23,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/264f3b1854404e1a8de0ab7ff1117d51 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/264f3b1854404e1a8de0ab7ff1117d51 2024-11-19T00:14:23,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/264f3b1854404e1a8de0ab7ff1117d51, entries=150, sequenceid=49, filesize=11.7 K 2024-11-19T00:14:23,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/3eb2cc4269514444b3520a23ea2974c3 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/3eb2cc4269514444b3520a23ea2974c3 2024-11-19T00:14:23,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/3eb2cc4269514444b3520a23ea2974c3, entries=150, sequenceid=49, filesize=11.7 K 2024-11-19T00:14:23,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for c24634f4d3f0bb64baf797330c06cc91 in 659ms, sequenceid=49, compaction requested=false 2024-11-19T00:14:23,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:23,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c24634f4d3f0bb64baf797330c06cc91 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T00:14:23,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=A 2024-11-19T00:14:23,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:23,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=B 2024-11-19T00:14:23,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:23,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=C 2024-11-19T00:14:23,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:23,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/f790e812762345259e2ee48a9cd53740 is 50, key is test_row_0/A:col10/1731975263177/Put/seqid=0 2024-11-19T00:14:23,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T00:14:23,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:23,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:23,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:23,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:23,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:23,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742069_1245 (size=12001) 2024-11-19T00:14:23,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/f790e812762345259e2ee48a9cd53740 2024-11-19T00:14:23,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/99727e89f29c48fabc42ce946d7c1348 is 50, key is test_row_0/B:col10/1731975263177/Put/seqid=0 2024-11-19T00:14:23,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975323249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:23,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975323256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:23,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:23,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:23,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:23,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742070_1246 (size=12001) 2024-11-19T00:14:23,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975323364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:23,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:23,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:23,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:23,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975323364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:23,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:23,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:23,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:23,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T00:14:23,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:23,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:23,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:23,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:23,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:23,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T00:14:23,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:23,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:23,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:23,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:23,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:23,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:23,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975323574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:23,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:23,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975323574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:23,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T00:14:23,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:23,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:23,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:23,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:23,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:23,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/99727e89f29c48fabc42ce946d7c1348 2024-11-19T00:14:23,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/d6034cf499384a2aaf2d986912db4bb0 is 50, key is test_row_0/C:col10/1731975263177/Put/seqid=0 2024-11-19T00:14:23,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742071_1247 (size=12001) 2024-11-19T00:14:23,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/d6034cf499384a2aaf2d986912db4bb0 2024-11-19T00:14:23,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/f790e812762345259e2ee48a9cd53740 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/f790e812762345259e2ee48a9cd53740 2024-11-19T00:14:23,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/f790e812762345259e2ee48a9cd53740, entries=150, sequenceid=61, filesize=11.7 K 2024-11-19T00:14:23,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/99727e89f29c48fabc42ce946d7c1348 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/99727e89f29c48fabc42ce946d7c1348 2024-11-19T00:14:23,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/99727e89f29c48fabc42ce946d7c1348, entries=150, sequenceid=61, filesize=11.7 K 2024-11-19T00:14:23,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/d6034cf499384a2aaf2d986912db4bb0 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/d6034cf499384a2aaf2d986912db4bb0 2024-11-19T00:14:23,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/d6034cf499384a2aaf2d986912db4bb0, entries=150, sequenceid=61, filesize=11.7 K 2024-11-19T00:14:23,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c24634f4d3f0bb64baf797330c06cc91 in 616ms, sequenceid=61, compaction requested=true 2024-11-19T00:14:23,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:23,797 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:23,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:14:23,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:23,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:14:23,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:23,797 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:23,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:14:23,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:23,798 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:23,798 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:23,799 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/B is initiating minor compaction (all files) 2024-11-19T00:14:23,799 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/A is initiating minor compaction (all files) 2024-11-19T00:14:23,799 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/A in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:23,799 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/B in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:23,799 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/ecce0326ec564a7e95be9eb28e5e50a5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/23fcc7b6131a4af69dc5284cb5655190, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/f790e812762345259e2ee48a9cd53740] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=35.2 K 2024-11-19T00:14:23,799 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/1186e2e1d1f446a0874387e6eb317576, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/264f3b1854404e1a8de0ab7ff1117d51, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/99727e89f29c48fabc42ce946d7c1348] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=35.2 K 2024-11-19T00:14:23,799 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1186e2e1d1f446a0874387e6eb317576, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731975261837 2024-11-19T00:14:23,800 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting ecce0326ec564a7e95be9eb28e5e50a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731975261837 2024-11-19T00:14:23,800 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 264f3b1854404e1a8de0ab7ff1117d51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731975261912 2024-11-19T00:14:23,800 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 23fcc7b6131a4af69dc5284cb5655190, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731975261912 2024-11-19T00:14:23,801 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99727e89f29c48fabc42ce946d7c1348, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731975262526 2024-11-19T00:14:23,801 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting f790e812762345259e2ee48a9cd53740, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731975262526 2024-11-19T00:14:23,810 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#A#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:23,810 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#B#compaction#202 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:23,810 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/1778d28317654c8ebb091cc8f1c14cbb is 50, key is test_row_0/A:col10/1731975263177/Put/seqid=0 2024-11-19T00:14:23,810 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/e9b8beb74bb24d3781354eae3949123e is 50, key is test_row_0/B:col10/1731975263177/Put/seqid=0 2024-11-19T00:14:23,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T00:14:23,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:23,836 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing c24634f4d3f0bb64baf797330c06cc91 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T00:14:23,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=A 2024-11-19T00:14:23,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:23,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=B 2024-11-19T00:14:23,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:23,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=C 2024-11-19T00:14:23,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:23,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742072_1248 (size=12104) 2024-11-19T00:14:23,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742073_1249 (size=12104) 2024-11-19T00:14:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:23,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:23,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/d7264b9a285045418442df726ab991af is 50, key is test_row_0/A:col10/1731975263245/Put/seqid=0 2024-11-19T00:14:23,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975323909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:23,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:23,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:23,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:23,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:23,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742074_1250 (size=12001) 2024-11-19T00:14:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975323915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:23,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:23,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:23,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:23,922 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/d7264b9a285045418442df726ab991af 2024-11-19T00:14:23,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/ef7293c84cac40ba809b656e50b425c5 is 50, key is test_row_0/B:col10/1731975263245/Put/seqid=0 2024-11-19T00:14:23,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742075_1251 (size=12001) 2024-11-19T00:14:23,959 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/ef7293c84cac40ba809b656e50b425c5 2024-11-19T00:14:23,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/dbaa71148d094e699b565b4cc8b0df9f is 50, key is test_row_0/C:col10/1731975263245/Put/seqid=0 2024-11-19T00:14:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T00:14:24,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742076_1252 (size=12001) 2024-11-19T00:14:24,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975324023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:24,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975324023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:24,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51968 deadline: 1731975324084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:24,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51992 deadline: 1731975324085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:24,086 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_1' in c24634f4d3f0bb64baf797330c06cc91 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:14:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:24,086 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_2' in c24634f4d3f0bb64baf797330c06cc91 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:14:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:24,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:24,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1731975324085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:24,094 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_1' in c24634f4d3f0bb64baf797330c06cc91 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:14:24,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:24,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975324234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:24,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975324234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:24,252 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/1778d28317654c8ebb091cc8f1c14cbb as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/1778d28317654c8ebb091cc8f1c14cbb 2024-11-19T00:14:24,263 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/A of c24634f4d3f0bb64baf797330c06cc91 into 1778d28317654c8ebb091cc8f1c14cbb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:24,263 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:24,263 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/A, priority=13, startTime=1731975263797; duration=0sec 2024-11-19T00:14:24,263 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:24,263 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:A 2024-11-19T00:14:24,263 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:24,265 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:24,265 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/C is initiating minor compaction (all files) 2024-11-19T00:14:24,266 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/C in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:24,266 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/37dcd6b9f793442184c6e5a4791f4477, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/3eb2cc4269514444b3520a23ea2974c3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/d6034cf499384a2aaf2d986912db4bb0] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=35.2 K 2024-11-19T00:14:24,270 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 37dcd6b9f793442184c6e5a4791f4477, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731975261837 2024-11-19T00:14:24,271 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eb2cc4269514444b3520a23ea2974c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731975261912 2024-11-19T00:14:24,271 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting d6034cf499384a2aaf2d986912db4bb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731975262526 2024-11-19T00:14:24,286 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/e9b8beb74bb24d3781354eae3949123e as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/e9b8beb74bb24d3781354eae3949123e 2024-11-19T00:14:24,292 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#C#compaction#206 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:24,292 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/439b1b13dca248df95b5c84ec7237ffe is 50, key is test_row_0/C:col10/1731975263177/Put/seqid=0 2024-11-19T00:14:24,302 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/B of c24634f4d3f0bb64baf797330c06cc91 into e9b8beb74bb24d3781354eae3949123e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:24,302 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:24,302 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/B, priority=13, startTime=1731975263797; duration=0sec 2024-11-19T00:14:24,302 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:24,302 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:B 2024-11-19T00:14:24,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742077_1253 (size=12104) 2024-11-19T00:14:24,339 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/439b1b13dca248df95b5c84ec7237ffe as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/439b1b13dca248df95b5c84ec7237ffe 2024-11-19T00:14:24,347 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/C of c24634f4d3f0bb64baf797330c06cc91 into 439b1b13dca248df95b5c84ec7237ffe(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:24,347 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:24,347 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/C, priority=13, startTime=1731975263797; duration=0sec 2024-11-19T00:14:24,347 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:24,347 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:C 2024-11-19T00:14:24,423 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/dbaa71148d094e699b565b4cc8b0df9f 2024-11-19T00:14:24,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/d7264b9a285045418442df726ab991af as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/d7264b9a285045418442df726ab991af 2024-11-19T00:14:24,446 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/d7264b9a285045418442df726ab991af, entries=150, sequenceid=81, filesize=11.7 K 2024-11-19T00:14:24,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/ef7293c84cac40ba809b656e50b425c5 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/ef7293c84cac40ba809b656e50b425c5 2024-11-19T00:14:24,454 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/ef7293c84cac40ba809b656e50b425c5, entries=150, sequenceid=81, filesize=11.7 K 2024-11-19T00:14:24,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/dbaa71148d094e699b565b4cc8b0df9f as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/dbaa71148d094e699b565b4cc8b0df9f 2024-11-19T00:14:24,460 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/dbaa71148d094e699b565b4cc8b0df9f, entries=150, sequenceid=81, filesize=11.7 K 2024-11-19T00:14:24,461 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c24634f4d3f0bb64baf797330c06cc91 in 624ms, sequenceid=81, compaction requested=false 2024-11-19T00:14:24,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:24,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:24,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-19T00:14:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-19T00:14:24,466 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-19T00:14:24,466 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6370 sec 2024-11-19T00:14:24,468 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.6450 sec 2024-11-19T00:14:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:24,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c24634f4d3f0bb64baf797330c06cc91 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T00:14:24,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=A 2024-11-19T00:14:24,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:24,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=B 2024-11-19T00:14:24,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:24,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=C 2024-11-19T00:14:24,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:24,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/b6b9beafd5164293ba434e29c9a6a73a is 50, key is test_row_0/A:col10/1731975264545/Put/seqid=0 2024-11-19T00:14:24,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742078_1254 (size=14341) 2024-11-19T00:14:24,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/b6b9beafd5164293ba434e29c9a6a73a 2024-11-19T00:14:24,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/333bf225b39041368b2fa4bc1d3eb78f is 50, key is test_row_0/B:col10/1731975264545/Put/seqid=0 2024-11-19T00:14:24,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975324594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:24,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742079_1255 (size=12001) 2024-11-19T00:14:24,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975324599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:24,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/333bf225b39041368b2fa4bc1d3eb78f 2024-11-19T00:14:24,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:24,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/ac12de8fac1041b1b1cf1fd9732d8077 is 50, key is test_row_0/C:col10/1731975264545/Put/seqid=0 2024-11-19T00:14:24,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742080_1256 (size=12001) 2024-11-19T00:14:24,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/ac12de8fac1041b1b1cf1fd9732d8077 2024-11-19T00:14:24,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/b6b9beafd5164293ba434e29c9a6a73a as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/b6b9beafd5164293ba434e29c9a6a73a 2024-11-19T00:14:24,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/b6b9beafd5164293ba434e29c9a6a73a, entries=200, sequenceid=98, filesize=14.0 K 2024-11-19T00:14:24,684 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-19T00:14:24,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/333bf225b39041368b2fa4bc1d3eb78f as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/333bf225b39041368b2fa4bc1d3eb78f 2024-11-19T00:14:24,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/333bf225b39041368b2fa4bc1d3eb78f, entries=150, sequenceid=98, filesize=11.7 K 2024-11-19T00:14:24,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/ac12de8fac1041b1b1cf1fd9732d8077 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/ac12de8fac1041b1b1cf1fd9732d8077 2024-11-19T00:14:24,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/ac12de8fac1041b1b1cf1fd9732d8077, entries=150, sequenceid=98, filesize=11.7 K 2024-11-19T00:14:24,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c24634f4d3f0bb64baf797330c06cc91 in 155ms, sequenceid=98, compaction requested=true 2024-11-19T00:14:24,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:24,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:14:24,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:24,702 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:24,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:14:24,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:24,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:14:24,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T00:14:24,702 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:24,703 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:24,703 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/A is initiating minor compaction (all files) 2024-11-19T00:14:24,703 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/A in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:24,703 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/1778d28317654c8ebb091cc8f1c14cbb, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/d7264b9a285045418442df726ab991af, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/b6b9beafd5164293ba434e29c9a6a73a] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=37.5 K 2024-11-19T00:14:24,704 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:24,704 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/B is initiating minor compaction (all files) 2024-11-19T00:14:24,704 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/B in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:24,704 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/e9b8beb74bb24d3781354eae3949123e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/ef7293c84cac40ba809b656e50b425c5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/333bf225b39041368b2fa4bc1d3eb78f] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=35.3 K 2024-11-19T00:14:24,705 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 1778d28317654c8ebb091cc8f1c14cbb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731975262526 2024-11-19T00:14:24,705 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9b8beb74bb24d3781354eae3949123e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731975262526 2024-11-19T00:14:24,705 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting d7264b9a285045418442df726ab991af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731975263229 2024-11-19T00:14:24,708 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef7293c84cac40ba809b656e50b425c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731975263229 2024-11-19T00:14:24,708 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting b6b9beafd5164293ba434e29c9a6a73a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731975263907 2024-11-19T00:14:24,709 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 333bf225b39041368b2fa4bc1d3eb78f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731975263913 2024-11-19T00:14:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:24,718 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c24634f4d3f0bb64baf797330c06cc91 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T00:14:24,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=A 2024-11-19T00:14:24,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:24,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=B 2024-11-19T00:14:24,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:24,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=C 2024-11-19T00:14:24,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:24,727 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#A#compaction#210 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:24,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/0fb766752b8a48b590ba3908d9d9595c is 50, key is test_row_0/A:col10/1731975264592/Put/seqid=0 2024-11-19T00:14:24,728 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/7b88b524b86d4ff7959f6bcd595b14f5 is 50, key is test_row_0/A:col10/1731975264545/Put/seqid=0 2024-11-19T00:14:24,732 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#B#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:24,733 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/13f79f830f544e50b7dccc016b2352eb is 50, key is test_row_0/B:col10/1731975264545/Put/seqid=0 2024-11-19T00:14:24,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742083_1259 (size=12207) 2024-11-19T00:14:24,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975324754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:24,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975324758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:24,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742081_1257 (size=12207) 2024-11-19T00:14:24,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742082_1258 (size=12001) 2024-11-19T00:14:24,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/0fb766752b8a48b590ba3908d9d9595c 2024-11-19T00:14:24,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/59d3b08475fd4877b38b93867e7c8164 is 50, key is test_row_0/B:col10/1731975264592/Put/seqid=0 2024-11-19T00:14:24,804 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T00:14:24,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742084_1260 (size=12001) 2024-11-19T00:14:24,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975324864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:24,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975324874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:25,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:25,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975325074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:25,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:25,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975325093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:25,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:25,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:25,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:25,170 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/13f79f830f544e50b7dccc016b2352eb as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/13f79f830f544e50b7dccc016b2352eb 2024-11-19T00:14:25,182 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/7b88b524b86d4ff7959f6bcd595b14f5 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/7b88b524b86d4ff7959f6bcd595b14f5 2024-11-19T00:14:25,186 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/B of c24634f4d3f0bb64baf797330c06cc91 into 13f79f830f544e50b7dccc016b2352eb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:25,186 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:25,186 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/B, priority=13, startTime=1731975264702; duration=0sec 2024-11-19T00:14:25,186 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:25,186 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:B 2024-11-19T00:14:25,186 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:25,187 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:25,188 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/C is initiating minor compaction (all files) 2024-11-19T00:14:25,188 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/C in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:25,188 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/439b1b13dca248df95b5c84ec7237ffe, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/dbaa71148d094e699b565b4cc8b0df9f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/ac12de8fac1041b1b1cf1fd9732d8077] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=35.3 K 2024-11-19T00:14:25,189 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 439b1b13dca248df95b5c84ec7237ffe, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731975262526 2024-11-19T00:14:25,190 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/A of c24634f4d3f0bb64baf797330c06cc91 into 7b88b524b86d4ff7959f6bcd595b14f5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:25,190 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:25,190 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbaa71148d094e699b565b4cc8b0df9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731975263229 2024-11-19T00:14:25,190 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/A, priority=13, startTime=1731975264701; duration=0sec 2024-11-19T00:14:25,190 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:25,190 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:A 2024-11-19T00:14:25,190 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac12de8fac1041b1b1cf1fd9732d8077, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731975263913 2024-11-19T00:14:25,209 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#C#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:25,210 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/52a356004a474a42a72ef96327a0e62c is 50, key is test_row_0/C:col10/1731975264545/Put/seqid=0 2024-11-19T00:14:25,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742085_1261 (size=12207) 2024-11-19T00:14:25,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/59d3b08475fd4877b38b93867e7c8164 2024-11-19T00:14:25,263 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/52a356004a474a42a72ef96327a0e62c as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/52a356004a474a42a72ef96327a0e62c 2024-11-19T00:14:25,274 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/C of c24634f4d3f0bb64baf797330c06cc91 into 52a356004a474a42a72ef96327a0e62c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:25,274 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:25,274 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/C, priority=13, startTime=1731975264702; duration=0sec 2024-11-19T00:14:25,274 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:25,274 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:C 2024-11-19T00:14:25,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/35d61a2ff6954c83be82ce6ae9e64053 is 50, key is test_row_0/C:col10/1731975264592/Put/seqid=0 2024-11-19T00:14:25,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742086_1262 (size=12001) 2024-11-19T00:14:25,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:25,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975325384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:25,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:25,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975325404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:25,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:25,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:25,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:25,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/35d61a2ff6954c83be82ce6ae9e64053 2024-11-19T00:14:25,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/0fb766752b8a48b590ba3908d9d9595c as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/0fb766752b8a48b590ba3908d9d9595c 2024-11-19T00:14:25,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/0fb766752b8a48b590ba3908d9d9595c, entries=150, sequenceid=117, filesize=11.7 K 2024-11-19T00:14:25,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/59d3b08475fd4877b38b93867e7c8164 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/59d3b08475fd4877b38b93867e7c8164 2024-11-19T00:14:25,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/59d3b08475fd4877b38b93867e7c8164, entries=150, sequenceid=117, filesize=11.7 K 2024-11-19T00:14:25,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/35d61a2ff6954c83be82ce6ae9e64053 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/35d61a2ff6954c83be82ce6ae9e64053 2024-11-19T00:14:25,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/35d61a2ff6954c83be82ce6ae9e64053, entries=150, sequenceid=117, filesize=11.7 K 2024-11-19T00:14:25,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c24634f4d3f0bb64baf797330c06cc91 in 1014ms, sequenceid=117, compaction requested=false 2024-11-19T00:14:25,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:25,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:25,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c24634f4d3f0bb64baf797330c06cc91 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T00:14:25,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=A 2024-11-19T00:14:25,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:25,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=B 2024-11-19T00:14:25,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:25,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=C 2024-11-19T00:14:25,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:25,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/3d4379e470774227aca756a75b2136a3 is 50, key is test_row_0/A:col10/1731975264750/Put/seqid=0 2024-11-19T00:14:25,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742087_1263 (size=14441) 2024-11-19T00:14:25,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:25,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975325953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:25,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:25,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:25,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:25,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:25,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975325958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:25,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:25,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:25,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T00:14:26,014 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:14:26,015 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:14:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-19T00:14:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T00:14:26,018 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:14:26,019 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:14:26,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:14:26,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:26,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975326063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:26,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:26,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:26,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:26,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975326074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:26,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51968 deadline: 1731975326094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:26,098 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_1' in c24634f4d3f0bb64baf797330c06cc91 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:14:26,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:26,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:26,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:26,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51992 deadline: 1731975326104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:26,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1731975326104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:26,108 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_2' in c24634f4d3f0bb64baf797330c06cc91 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:14:26,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:26,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:26,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:26,109 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_1' in c24634f4d3f0bb64baf797330c06cc91 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:14:26,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:26,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:26,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T00:14:26,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T00:14:26,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:26,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:26,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:26,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:26,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:26,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975326274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:26,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:26,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:26,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:26,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975326283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:26,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:26,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:26,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:26,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T00:14:26,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:26,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:26,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:26,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:26,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:26,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/3d4379e470774227aca756a75b2136a3 2024-11-19T00:14:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T00:14:26,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/cedead13538545f9b1778ea190ac714b is 50, key is test_row_0/B:col10/1731975264750/Put/seqid=0 2024-11-19T00:14:26,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742088_1264 (size=12051) 2024-11-19T00:14:26,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/cedead13538545f9b1778ea190ac714b 2024-11-19T00:14:26,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/9a919190a442441a8f6d2596dafe2e59 is 50, key is test_row_0/C:col10/1731975264750/Put/seqid=0 2024-11-19T00:14:26,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742089_1265 (size=12051) 2024-11-19T00:14:26,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T00:14:26,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:26,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:26,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:26,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:26,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:26,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975326583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:26,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:26,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:26,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:26,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975326594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:26,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:26,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:26,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:26,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T00:14:26,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:26,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:26,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:26,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:26,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T00:14:26,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/9a919190a442441a8f6d2596dafe2e59 2024-11-19T00:14:26,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/3d4379e470774227aca756a75b2136a3 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/3d4379e470774227aca756a75b2136a3 2024-11-19T00:14:26,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/3d4379e470774227aca756a75b2136a3, entries=200, sequenceid=134, filesize=14.1 K 2024-11-19T00:14:26,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/cedead13538545f9b1778ea190ac714b as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/cedead13538545f9b1778ea190ac714b 2024-11-19T00:14:26,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T00:14:26,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:26,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:26,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:26,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:26,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:26,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/cedead13538545f9b1778ea190ac714b, entries=150, sequenceid=134, filesize=11.8 K 2024-11-19T00:14:26,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/9a919190a442441a8f6d2596dafe2e59 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/9a919190a442441a8f6d2596dafe2e59 2024-11-19T00:14:26,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/9a919190a442441a8f6d2596dafe2e59, entries=150, sequenceid=134, filesize=11.8 K 2024-11-19T00:14:26,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c24634f4d3f0bb64baf797330c06cc91 in 898ms, sequenceid=134, compaction requested=true 2024-11-19T00:14:26,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:26,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:14:26,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:26,794 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:26,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:14:26,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:26,794 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:26,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:14:26,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:26,795 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:26,795 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38649 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:26,795 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/B is initiating minor compaction (all files) 2024-11-19T00:14:26,795 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/A is initiating minor compaction (all files) 2024-11-19T00:14:26,795 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/B in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:26,795 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/A in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:26,796 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/13f79f830f544e50b7dccc016b2352eb, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/59d3b08475fd4877b38b93867e7c8164, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/cedead13538545f9b1778ea190ac714b] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=35.4 K 2024-11-19T00:14:26,796 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/7b88b524b86d4ff7959f6bcd595b14f5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/0fb766752b8a48b590ba3908d9d9595c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/3d4379e470774227aca756a75b2136a3] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=37.7 K 2024-11-19T00:14:26,796 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 13f79f830f544e50b7dccc016b2352eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731975263913 2024-11-19T00:14:26,796 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b88b524b86d4ff7959f6bcd595b14f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731975263913 2024-11-19T00:14:26,796 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 59d3b08475fd4877b38b93867e7c8164, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731975264592 2024-11-19T00:14:26,796 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fb766752b8a48b590ba3908d9d9595c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731975264592 2024-11-19T00:14:26,796 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting cedead13538545f9b1778ea190ac714b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731975264730 2024-11-19T00:14:26,796 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d4379e470774227aca756a75b2136a3, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731975264730 2024-11-19T00:14:26,805 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#B#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:26,806 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/2e0e01dea1d741ec8e95966c872f9b83 is 50, key is test_row_0/B:col10/1731975264750/Put/seqid=0 2024-11-19T00:14:26,812 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#A#compaction#220 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:26,812 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/8a09ef74a64a42da8091933a61cc850b is 50, key is test_row_0/A:col10/1731975264750/Put/seqid=0 2024-11-19T00:14:26,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742090_1266 (size=12359) 2024-11-19T00:14:26,842 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/2e0e01dea1d741ec8e95966c872f9b83 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/2e0e01dea1d741ec8e95966c872f9b83 2024-11-19T00:14:26,850 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/B of c24634f4d3f0bb64baf797330c06cc91 into 2e0e01dea1d741ec8e95966c872f9b83(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:26,850 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:26,850 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/B, priority=13, startTime=1731975266794; duration=0sec 2024-11-19T00:14:26,851 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:26,851 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:B 2024-11-19T00:14:26,851 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:26,852 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:26,852 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/C is initiating minor compaction (all files) 2024-11-19T00:14:26,852 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/C in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:26,852 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/52a356004a474a42a72ef96327a0e62c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/35d61a2ff6954c83be82ce6ae9e64053, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/9a919190a442441a8f6d2596dafe2e59] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=35.4 K 2024-11-19T00:14:26,852 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 52a356004a474a42a72ef96327a0e62c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731975263913 2024-11-19T00:14:26,853 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 35d61a2ff6954c83be82ce6ae9e64053, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731975264592 2024-11-19T00:14:26,853 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a919190a442441a8f6d2596dafe2e59, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731975264730 2024-11-19T00:14:26,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742091_1267 (size=12359) 2024-11-19T00:14:26,868 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#C#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:26,868 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/b978b23e9915475ca4eb91419c2bb8d1 is 50, key is test_row_0/C:col10/1731975264750/Put/seqid=0 2024-11-19T00:14:26,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742092_1268 (size=12359) 2024-11-19T00:14:26,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T00:14:26,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:26,938 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing c24634f4d3f0bb64baf797330c06cc91 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T00:14:26,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=A 2024-11-19T00:14:26,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:26,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=B 2024-11-19T00:14:26,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:26,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=C 2024-11-19T00:14:26,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:26,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/9adf9c898ac74cd99e49bebecc20f31b is 50, key is test_row_0/A:col10/1731975265933/Put/seqid=0 2024-11-19T00:14:26,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742093_1269 (size=12151) 2024-11-19T00:14:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:27,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:27,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975327123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:27,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:27,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:27,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:27,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975327126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:27,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:27,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:27,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T00:14:27,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975327233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:27,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975327234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:27,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:27,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:27,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:27,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:27,271 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/8a09ef74a64a42da8091933a61cc850b as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/8a09ef74a64a42da8091933a61cc850b 2024-11-19T00:14:27,277 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/A of c24634f4d3f0bb64baf797330c06cc91 into 8a09ef74a64a42da8091933a61cc850b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:27,277 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:27,277 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/A, priority=13, startTime=1731975266794; duration=0sec 2024-11-19T00:14:27,277 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:27,277 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:A 2024-11-19T00:14:27,308 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/b978b23e9915475ca4eb91419c2bb8d1 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/b978b23e9915475ca4eb91419c2bb8d1 2024-11-19T00:14:27,314 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/C of c24634f4d3f0bb64baf797330c06cc91 into b978b23e9915475ca4eb91419c2bb8d1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:27,314 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:27,314 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/C, priority=13, startTime=1731975266794; duration=0sec 2024-11-19T00:14:27,314 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:27,314 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:C 2024-11-19T00:14:27,351 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/9adf9c898ac74cd99e49bebecc20f31b 2024-11-19T00:14:27,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/9188c81d457947f088fbdf48415eb2a4 is 50, key is test_row_0/B:col10/1731975265933/Put/seqid=0 2024-11-19T00:14:27,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742094_1270 (size=12151) 2024-11-19T00:14:27,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975327446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:27,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975327446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:27,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975327754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:27,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:27,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:27,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:27,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:27,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975327754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:27,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:27,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:27,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:27,764 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/9188c81d457947f088fbdf48415eb2a4 2024-11-19T00:14:27,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/80a2f833fa774711bbf06fae57e2905a is 50, key is test_row_0/C:col10/1731975265933/Put/seqid=0 2024-11-19T00:14:27,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742095_1271 (size=12151) 2024-11-19T00:14:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T00:14:28,189 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/80a2f833fa774711bbf06fae57e2905a 2024-11-19T00:14:28,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/9adf9c898ac74cd99e49bebecc20f31b as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/9adf9c898ac74cd99e49bebecc20f31b 2024-11-19T00:14:28,199 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/9adf9c898ac74cd99e49bebecc20f31b, entries=150, sequenceid=153, filesize=11.9 K 2024-11-19T00:14:28,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/9188c81d457947f088fbdf48415eb2a4 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/9188c81d457947f088fbdf48415eb2a4 2024-11-19T00:14:28,204 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/9188c81d457947f088fbdf48415eb2a4, entries=150, sequenceid=153, filesize=11.9 K 2024-11-19T00:14:28,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/80a2f833fa774711bbf06fae57e2905a as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/80a2f833fa774711bbf06fae57e2905a 2024-11-19T00:14:28,209 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/80a2f833fa774711bbf06fae57e2905a, entries=150, sequenceid=153, filesize=11.9 K 2024-11-19T00:14:28,210 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c24634f4d3f0bb64baf797330c06cc91 in 1272ms, sequenceid=153, compaction requested=false 2024-11-19T00:14:28,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:28,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:28,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-19T00:14:28,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-19T00:14:28,214 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-19T00:14:28,215 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1940 sec 2024-11-19T00:14:28,216 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.2000 sec 2024-11-19T00:14:28,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:28,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c24634f4d3f0bb64baf797330c06cc91 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T00:14:28,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=A 2024-11-19T00:14:28,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:28,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=B 2024-11-19T00:14:28,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:28,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=C 2024-11-19T00:14:28,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:28,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/42bd0e6b1dc543518926eac6e0b57174 is 50, key is test_row_0/A:col10/1731975268265/Put/seqid=0 2024-11-19T00:14:28,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742096_1272 (size=14541) 2024-11-19T00:14:28,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975328310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:28,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975328311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:28,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975328424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:28,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975328424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:28,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975328634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:28,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:28,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:28,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:28,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975328635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:28,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:28,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:28,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:28,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/42bd0e6b1dc543518926eac6e0b57174 2024-11-19T00:14:28,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/b1f8666a3b7b4cdda30046a836bb1562 is 50, key is test_row_0/B:col10/1731975268265/Put/seqid=0 2024-11-19T00:14:28,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742097_1273 (size=12151) 2024-11-19T00:14:28,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:28,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975328944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:28,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:28,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:28,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975328944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:28,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:28,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:29,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/b1f8666a3b7b4cdda30046a836bb1562 2024-11-19T00:14:29,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/a03d87df29394e2bb0173629de4fec3b is 50, key is test_row_0/C:col10/1731975268265/Put/seqid=0 2024-11-19T00:14:29,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742098_1274 (size=12151) 2024-11-19T00:14:29,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/a03d87df29394e2bb0173629de4fec3b 2024-11-19T00:14:29,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/42bd0e6b1dc543518926eac6e0b57174 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/42bd0e6b1dc543518926eac6e0b57174 2024-11-19T00:14:29,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/42bd0e6b1dc543518926eac6e0b57174, entries=200, sequenceid=170, filesize=14.2 K 2024-11-19T00:14:29,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/b1f8666a3b7b4cdda30046a836bb1562 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/b1f8666a3b7b4cdda30046a836bb1562 2024-11-19T00:14:29,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/b1f8666a3b7b4cdda30046a836bb1562, entries=150, sequenceid=170, filesize=11.9 K 2024-11-19T00:14:29,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/a03d87df29394e2bb0173629de4fec3b as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/a03d87df29394e2bb0173629de4fec3b 2024-11-19T00:14:29,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/a03d87df29394e2bb0173629de4fec3b, entries=150, sequenceid=170, filesize=11.9 K 2024-11-19T00:14:29,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for c24634f4d3f0bb64baf797330c06cc91 in 926ms, sequenceid=170, compaction requested=true 2024-11-19T00:14:29,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:29,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:14:29,192 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:29,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:29,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:14:29,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:29,192 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:29,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:14:29,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:29,195 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:29,195 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/B is initiating minor compaction (all files) 2024-11-19T00:14:29,195 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39051 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:29,195 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/A is initiating minor compaction (all files) 2024-11-19T00:14:29,195 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/B in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:29,195 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/A in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:29,195 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/2e0e01dea1d741ec8e95966c872f9b83, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/9188c81d457947f088fbdf48415eb2a4, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/b1f8666a3b7b4cdda30046a836bb1562] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=35.8 K 2024-11-19T00:14:29,195 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/8a09ef74a64a42da8091933a61cc850b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/9adf9c898ac74cd99e49bebecc20f31b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/42bd0e6b1dc543518926eac6e0b57174] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=38.1 K 2024-11-19T00:14:29,195 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a09ef74a64a42da8091933a61cc850b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731975264730 2024-11-19T00:14:29,196 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e0e01dea1d741ec8e95966c872f9b83, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731975264730 2024-11-19T00:14:29,196 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9adf9c898ac74cd99e49bebecc20f31b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731975265933 2024-11-19T00:14:29,196 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 9188c81d457947f088fbdf48415eb2a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731975265933 2024-11-19T00:14:29,196 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting b1f8666a3b7b4cdda30046a836bb1562, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731975267122 2024-11-19T00:14:29,196 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42bd0e6b1dc543518926eac6e0b57174, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731975267118 2024-11-19T00:14:29,205 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#B#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:29,205 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/bc35b1d3aac14aa8bb57c94f11a90739 is 50, key is test_row_0/B:col10/1731975268265/Put/seqid=0 2024-11-19T00:14:29,212 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#A#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:29,212 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/66b45029a71d483b8de47a6788f1db37 is 50, key is test_row_0/A:col10/1731975268265/Put/seqid=0 2024-11-19T00:14:29,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742099_1275 (size=12561) 2024-11-19T00:14:29,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742100_1276 (size=12561) 2024-11-19T00:14:29,220 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/bc35b1d3aac14aa8bb57c94f11a90739 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/bc35b1d3aac14aa8bb57c94f11a90739 2024-11-19T00:14:29,229 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/B of c24634f4d3f0bb64baf797330c06cc91 into bc35b1d3aac14aa8bb57c94f11a90739(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:29,229 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:29,229 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/B, priority=13, startTime=1731975269192; duration=0sec 2024-11-19T00:14:29,229 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:29,229 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:B 2024-11-19T00:14:29,229 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:29,231 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:29,231 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/C is initiating minor compaction (all files) 2024-11-19T00:14:29,231 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/C in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:29,231 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/b978b23e9915475ca4eb91419c2bb8d1, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/80a2f833fa774711bbf06fae57e2905a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/a03d87df29394e2bb0173629de4fec3b] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=35.8 K 2024-11-19T00:14:29,231 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting b978b23e9915475ca4eb91419c2bb8d1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731975264730 2024-11-19T00:14:29,232 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 80a2f833fa774711bbf06fae57e2905a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731975265933 2024-11-19T00:14:29,232 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting a03d87df29394e2bb0173629de4fec3b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731975267122 2024-11-19T00:14:29,240 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#C#compaction#230 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:29,241 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/44a04929820f4aacbcf9555fd209fefc is 50, key is test_row_0/C:col10/1731975268265/Put/seqid=0 2024-11-19T00:14:29,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742101_1277 (size=12561) 2024-11-19T00:14:29,283 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/44a04929820f4aacbcf9555fd209fefc as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/44a04929820f4aacbcf9555fd209fefc 2024-11-19T00:14:29,288 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/C of c24634f4d3f0bb64baf797330c06cc91 into 44a04929820f4aacbcf9555fd209fefc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:29,289 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:29,289 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/C, priority=13, startTime=1731975269192; duration=0sec 2024-11-19T00:14:29,289 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:29,289 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:C 2024-11-19T00:14:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:29,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c24634f4d3f0bb64baf797330c06cc91 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T00:14:29,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=A 2024-11-19T00:14:29,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:29,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=B 2024-11-19T00:14:29,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:29,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=C 2024-11-19T00:14:29,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:29,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/c9a86e7635c2471c9430097427f3c4dc is 50, key is test_row_0/A:col10/1731975269456/Put/seqid=0 2024-11-19T00:14:29,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742102_1278 (size=16931) 2024-11-19T00:14:29,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975329482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:29,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:29,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:29,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:29,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975329486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:29,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:29,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:29,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:29,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975329593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:29,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:29,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:29,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:29,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975329593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:29,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:29,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:29,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:29,626 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/66b45029a71d483b8de47a6788f1db37 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/66b45029a71d483b8de47a6788f1db37 2024-11-19T00:14:29,631 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/A of c24634f4d3f0bb64baf797330c06cc91 into 66b45029a71d483b8de47a6788f1db37(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:29,631 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:29,631 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/A, priority=13, startTime=1731975269192; duration=0sec 2024-11-19T00:14:29,631 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:29,631 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:A 2024-11-19T00:14:29,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:29,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975329804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:29,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:29,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975329804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:29,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:29,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:29,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:29,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:29,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:29,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:29,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/c9a86e7635c2471c9430097427f3c4dc 2024-11-19T00:14:29,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/81d5fdca304a41a2869f6a23bdff69f8 is 50, key is test_row_0/B:col10/1731975269456/Put/seqid=0 2024-11-19T00:14:29,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742103_1279 (size=12151) 2024-11-19T00:14:30,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:30,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975330113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:30,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:30,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:30,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:30,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:30,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:30,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51968 deadline: 1731975330116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:30,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975330116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:30,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:30,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:30,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:30,120 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_1' in c24634f4d3f0bb64baf797330c06cc91 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:14:30,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:30,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:30,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:30,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51992 deadline: 1731975330144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:30,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1731975330144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:30,147 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_1' in c24634f4d3f0bb64baf797330c06cc91 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:14:30,147 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_2' in c24634f4d3f0bb64baf797330c06cc91 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:14:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T00:14:30,184 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:14:30,184 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:14:30,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-19T00:14:30,186 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:14:30,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T00:14:30,187 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:14:30,187 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:14:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T00:14:30,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/81d5fdca304a41a2869f6a23bdff69f8 2024-11-19T00:14:30,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/5b84d6fc9573419aad9d7c5b1c8a8658 is 50, key is test_row_0/C:col10/1731975269456/Put/seqid=0 2024-11-19T00:14:30,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742104_1280 (size=12151) 2024-11-19T00:14:30,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/5b84d6fc9573419aad9d7c5b1c8a8658 2024-11-19T00:14:30,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/c9a86e7635c2471c9430097427f3c4dc as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/c9a86e7635c2471c9430097427f3c4dc 2024-11-19T00:14:30,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/c9a86e7635c2471c9430097427f3c4dc, entries=250, sequenceid=192, filesize=16.5 K 2024-11-19T00:14:30,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/81d5fdca304a41a2869f6a23bdff69f8 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/81d5fdca304a41a2869f6a23bdff69f8 2024-11-19T00:14:30,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/81d5fdca304a41a2869f6a23bdff69f8, entries=150, sequenceid=192, filesize=11.9 K 2024-11-19T00:14:30,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/5b84d6fc9573419aad9d7c5b1c8a8658 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/5b84d6fc9573419aad9d7c5b1c8a8658 2024-11-19T00:14:30,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T00:14:30,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:30,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:30,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:30,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:30,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:30,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/5b84d6fc9573419aad9d7c5b1c8a8658, entries=150, sequenceid=192, filesize=11.9 K 2024-11-19T00:14:30,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c24634f4d3f0bb64baf797330c06cc91 in 892ms, sequenceid=192, compaction requested=false 2024-11-19T00:14:30,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:30,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T00:14:30,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:30,493 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing c24634f4d3f0bb64baf797330c06cc91 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T00:14:30,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=A 2024-11-19T00:14:30,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:30,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=B 2024-11-19T00:14:30,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:30,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=C 2024-11-19T00:14:30,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:30,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/602b410387f3464a8a166fec614ae9fc is 50, key is test_row_0/A:col10/1731975269480/Put/seqid=0 2024-11-19T00:14:30,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742105_1281 (size=9757) 2024-11-19T00:14:30,503 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/602b410387f3464a8a166fec614ae9fc 2024-11-19T00:14:30,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T00:14:30,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/d3c41270738e43ef9c69652b620e8b94 is 50, key is test_row_0/B:col10/1731975269480/Put/seqid=0 2024-11-19T00:14:30,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742106_1282 (size=9757) 2024-11-19T00:14:30,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:30,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:30,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975330667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:30,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:30,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:30,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:30,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975330669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:30,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:30,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975330774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:30,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:30,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975330784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:30,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:30,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:30,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:30,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T00:14:30,920 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/d3c41270738e43ef9c69652b620e8b94 2024-11-19T00:14:30,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/09e5e74a037848d1993b01eeef08d996 is 50, key is test_row_0/C:col10/1731975269480/Put/seqid=0 2024-11-19T00:14:30,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742107_1283 (size=9757) 2024-11-19T00:14:30,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975330984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:30,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:30,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:30,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:30,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975330993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:31,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975331294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:31,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975331303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:31,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:31,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:31,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:31,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T00:14:31,341 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/09e5e74a037848d1993b01eeef08d996 2024-11-19T00:14:31,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/602b410387f3464a8a166fec614ae9fc as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/602b410387f3464a8a166fec614ae9fc 2024-11-19T00:14:31,354 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/602b410387f3464a8a166fec614ae9fc, entries=100, sequenceid=205, filesize=9.5 K 2024-11-19T00:14:31,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/d3c41270738e43ef9c69652b620e8b94 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/d3c41270738e43ef9c69652b620e8b94 2024-11-19T00:14:31,361 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/d3c41270738e43ef9c69652b620e8b94, entries=100, sequenceid=205, filesize=9.5 K 2024-11-19T00:14:31,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/09e5e74a037848d1993b01eeef08d996 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/09e5e74a037848d1993b01eeef08d996 2024-11-19T00:14:31,368 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/09e5e74a037848d1993b01eeef08d996, entries=100, sequenceid=205, filesize=9.5 K 2024-11-19T00:14:31,369 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c24634f4d3f0bb64baf797330c06cc91 in 876ms, sequenceid=205, compaction requested=true 2024-11-19T00:14:31,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:31,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:31,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-19T00:14:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-19T00:14:31,374 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-19T00:14:31,374 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1840 sec 2024-11-19T00:14:31,376 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.1910 sec 2024-11-19T00:14:31,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c24634f4d3f0bb64baf797330c06cc91 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-19T00:14:31,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=A 2024-11-19T00:14:31,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:14:31,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=B 2024-11-19T00:14:31,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:14:31,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=C 2024-11-19T00:14:31,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:14:31,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:31,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/f3b53aa8df4c4f03a667d2ad77e68d31 is 50, key is test_row_0/A:col10/1731975270668/Put/seqid=0 2024-11-19T00:14:31,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742108_1284 (size=12151) 2024-11-19T00:14:31,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/f3b53aa8df4c4f03a667d2ad77e68d31 2024-11-19T00:14:31,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/5da538c1c4cd4b68b8b2934b57868f5e is 50, key is test_row_0/B:col10/1731975270668/Put/seqid=0 2024-11-19T00:14:31,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:31,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975331836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:31,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:31,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:31,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:31,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:31,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975331839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:31,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:31,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:31,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:31,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742109_1285 (size=12151) 2024-11-19T00:14:31,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:31,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975331944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:31,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:31,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:31,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:31,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:31,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975331954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:31,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:31,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:31,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:32,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:32,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975332153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:32,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:32,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:32,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:32,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:32,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975332164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:32,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:32,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:32,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:32,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/5da538c1c4cd4b68b8b2934b57868f5e 2024-11-19T00:14:32,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/f8850ddfa0884be8b78c0ec74526c279 is 50, key is test_row_0/C:col10/1731975270668/Put/seqid=0 2024-11-19T00:14:32,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742110_1286 (size=12151) 2024-11-19T00:14:32,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T00:14:32,333 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:14:32,334 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:14:32,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-19T00:14:32,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T00:14:32,347 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:14:32,348 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:14:32,348 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:14:32,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T00:14:32,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:32,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975332463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:32,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:32,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:32,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:32,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975332474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:32,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:32,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:32,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:32,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T00:14:32,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:32,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:32,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:32,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:32,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:32,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:32,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T00:14:32,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:32,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:32,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:32,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:32,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:32,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T00:14:32,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/f8850ddfa0884be8b78c0ec74526c279 2024-11-19T00:14:32,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/f3b53aa8df4c4f03a667d2ad77e68d31 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/f3b53aa8df4c4f03a667d2ad77e68d31 2024-11-19T00:14:32,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/f3b53aa8df4c4f03a667d2ad77e68d31, entries=150, sequenceid=226, filesize=11.9 K 2024-11-19T00:14:32,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/5da538c1c4cd4b68b8b2934b57868f5e as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/5da538c1c4cd4b68b8b2934b57868f5e 2024-11-19T00:14:32,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/5da538c1c4cd4b68b8b2934b57868f5e, entries=150, sequenceid=226, filesize=11.9 K 2024-11-19T00:14:32,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/f8850ddfa0884be8b78c0ec74526c279 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/f8850ddfa0884be8b78c0ec74526c279 2024-11-19T00:14:32,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/f8850ddfa0884be8b78c0ec74526c279, entries=150, sequenceid=226, filesize=11.9 K 2024-11-19T00:14:32,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for c24634f4d3f0bb64baf797330c06cc91 in 894ms, sequenceid=226, compaction requested=true 2024-11-19T00:14:32,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:32,704 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:14:32,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:14:32,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:32,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:14:32,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:32,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:14:32,704 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:14:32,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:32,712 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51400 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:14:32,712 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/A is initiating minor compaction (all files) 2024-11-19T00:14:32,712 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/A in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:32,712 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/66b45029a71d483b8de47a6788f1db37, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/c9a86e7635c2471c9430097427f3c4dc, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/602b410387f3464a8a166fec614ae9fc, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/f3b53aa8df4c4f03a667d2ad77e68d31] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=50.2 K 2024-11-19T00:14:32,712 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46620 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:14:32,713 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/B is initiating minor compaction (all files) 2024-11-19T00:14:32,713 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/B in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:32,713 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/bc35b1d3aac14aa8bb57c94f11a90739, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/81d5fdca304a41a2869f6a23bdff69f8, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/d3c41270738e43ef9c69652b620e8b94, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/5da538c1c4cd4b68b8b2934b57868f5e] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=45.5 K 2024-11-19T00:14:32,713 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66b45029a71d483b8de47a6788f1db37, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731975267122 2024-11-19T00:14:32,714 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting bc35b1d3aac14aa8bb57c94f11a90739, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731975267122 2024-11-19T00:14:32,714 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 81d5fdca304a41a2869f6a23bdff69f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731975268309 2024-11-19T00:14:32,714 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9a86e7635c2471c9430097427f3c4dc, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731975268303 2024-11-19T00:14:32,714 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting d3c41270738e43ef9c69652b620e8b94, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731975269480 2024-11-19T00:14:32,714 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 602b410387f3464a8a166fec614ae9fc, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731975269480 2024-11-19T00:14:32,715 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 5da538c1c4cd4b68b8b2934b57868f5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731975270666 2024-11-19T00:14:32,715 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3b53aa8df4c4f03a667d2ad77e68d31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731975270666 2024-11-19T00:14:32,728 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#B#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:32,728 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#A#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:32,729 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/dd437c6a14284541b5e9d5f94326f4bd is 50, key is test_row_0/A:col10/1731975270668/Put/seqid=0 2024-11-19T00:14:32,729 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/f7dbdf918fa54d76b6ce8bfc0b753614 is 50, key is test_row_0/B:col10/1731975270668/Put/seqid=0 2024-11-19T00:14:32,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742111_1287 (size=12697) 2024-11-19T00:14:32,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742112_1288 (size=12697) 2024-11-19T00:14:32,742 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/dd437c6a14284541b5e9d5f94326f4bd as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/dd437c6a14284541b5e9d5f94326f4bd 2024-11-19T00:14:32,749 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/A of c24634f4d3f0bb64baf797330c06cc91 into dd437c6a14284541b5e9d5f94326f4bd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:32,749 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:32,749 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/A, priority=12, startTime=1731975272703; duration=0sec 2024-11-19T00:14:32,749 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:32,749 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:A 2024-11-19T00:14:32,749 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:14:32,751 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46620 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:14:32,751 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/C is initiating minor compaction (all files) 2024-11-19T00:14:32,751 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/C in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:32,751 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/44a04929820f4aacbcf9555fd209fefc, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/5b84d6fc9573419aad9d7c5b1c8a8658, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/09e5e74a037848d1993b01eeef08d996, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/f8850ddfa0884be8b78c0ec74526c279] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=45.5 K 2024-11-19T00:14:32,752 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44a04929820f4aacbcf9555fd209fefc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731975267122 2024-11-19T00:14:32,752 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b84d6fc9573419aad9d7c5b1c8a8658, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731975268309 2024-11-19T00:14:32,752 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09e5e74a037848d1993b01eeef08d996, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731975269480 2024-11-19T00:14:32,752 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8850ddfa0884be8b78c0ec74526c279, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731975270666 2024-11-19T00:14:32,760 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#C#compaction#242 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:32,761 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/6cfc079d09304998b3aaae783145bae5 is 50, key is test_row_0/C:col10/1731975270668/Put/seqid=0 2024-11-19T00:14:32,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742113_1289 (size=12697) 2024-11-19T00:14:32,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T00:14:32,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:32,807 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing c24634f4d3f0bb64baf797330c06cc91 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T00:14:32,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=A 2024-11-19T00:14:32,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:32,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=B 2024-11-19T00:14:32,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:32,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=C 2024-11-19T00:14:32,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:32,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/c82c498f6dca49b392a21085aa886b00 is 50, key is test_row_0/A:col10/1731975271828/Put/seqid=0 2024-11-19T00:14:32,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742114_1290 (size=12151) 2024-11-19T00:14:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T00:14:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:32,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:33,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975273018, max timestamp=1731975273029Num uniques 859; 2024-11-19T00:14:33,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975273018, max timestamp=1731975273029Num uniques 859; 2024-11-19T00:14:33,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:14:33,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:14:33,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975273018, max timestamp=1731975273029Num uniques 859; 2024-11-19T00:14:33,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:14:33,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:33,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975333081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:33,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:33,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975333082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:33,139 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/f7dbdf918fa54d76b6ce8bfc0b753614 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/f7dbdf918fa54d76b6ce8bfc0b753614 2024-11-19T00:14:33,146 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/B of c24634f4d3f0bb64baf797330c06cc91 into f7dbdf918fa54d76b6ce8bfc0b753614(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:33,146 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:33,146 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/B, priority=12, startTime=1731975272704; duration=0sec 2024-11-19T00:14:33,146 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:33,146 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:B 2024-11-19T00:14:33,174 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/6cfc079d09304998b3aaae783145bae5 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/6cfc079d09304998b3aaae783145bae5 2024-11-19T00:14:33,179 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/C of c24634f4d3f0bb64baf797330c06cc91 into 6cfc079d09304998b3aaae783145bae5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:33,179 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:33,180 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/C, priority=12, startTime=1731975272704; duration=0sec 2024-11-19T00:14:33,180 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:33,180 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:C 2024-11-19T00:14:33,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:33,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975333193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:33,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:33,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975333193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:33,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:33,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:33,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:33,221 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/c82c498f6dca49b392a21085aa886b00 2024-11-19T00:14:33,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/9de960dc8a6d48809bb7361cf7a2fd89 is 50, key is test_row_0/B:col10/1731975271828/Put/seqid=0 2024-11-19T00:14:33,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742115_1291 (size=12151) 2024-11-19T00:14:33,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:33,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975333404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:33,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:33,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:33,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:33,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975333404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:33,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:33,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:33,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:33,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:33,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T00:14:33,633 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/9de960dc8a6d48809bb7361cf7a2fd89 2024-11-19T00:14:33,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/f411809aafb04580b87edb43f7d8d253 is 50, key is test_row_0/C:col10/1731975271828/Put/seqid=0 2024-11-19T00:14:33,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742116_1292 (size=12151) 2024-11-19T00:14:33,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975333714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:33,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975333714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:34,049 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/f411809aafb04580b87edb43f7d8d253 2024-11-19T00:14:34,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/c82c498f6dca49b392a21085aa886b00 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/c82c498f6dca49b392a21085aa886b00 2024-11-19T00:14:34,067 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/c82c498f6dca49b392a21085aa886b00, entries=150, sequenceid=238, filesize=11.9 K 2024-11-19T00:14:34,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/9de960dc8a6d48809bb7361cf7a2fd89 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/9de960dc8a6d48809bb7361cf7a2fd89 2024-11-19T00:14:34,071 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/9de960dc8a6d48809bb7361cf7a2fd89, entries=150, sequenceid=238, filesize=11.9 K 2024-11-19T00:14:34,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/f411809aafb04580b87edb43f7d8d253 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/f411809aafb04580b87edb43f7d8d253 2024-11-19T00:14:34,075 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/f411809aafb04580b87edb43f7d8d253, entries=150, sequenceid=238, filesize=11.9 K 2024-11-19T00:14:34,076 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c24634f4d3f0bb64baf797330c06cc91 in 1269ms, sequenceid=238, compaction requested=false 2024-11-19T00:14:34,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:34,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:34,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-19T00:14:34,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-19T00:14:34,080 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-19T00:14:34,080 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7300 sec 2024-11-19T00:14:34,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.7470 sec 2024-11-19T00:14:34,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:34,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c24634f4d3f0bb64baf797330c06cc91 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-19T00:14:34,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=A 2024-11-19T00:14:34,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:14:34,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=B 2024-11-19T00:14:34,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:14:34,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=C 2024-11-19T00:14:34,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:14:34,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/b654684f4b0d4df288130463d5124b91 is 50, key is test_row_0/A:col10/1731975273067/Put/seqid=0 2024-11-19T00:14:34,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742117_1293 (size=14741) 2024-11-19T00:14:34,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:34,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975334248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:34,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:34,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:34,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:34,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:34,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975334249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:34,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:34,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975334364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:34,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:34,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:34,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975334364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:34,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:34,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:34,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:34,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:34,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:34,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T00:14:34,494 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:14:34,494 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:14:34,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-19T00:14:34,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T00:14:34,496 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:14:34,497 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:14:34,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:14:34,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:34,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:34,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975334574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:34,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975334574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T00:14:34,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/b654684f4b0d4df288130463d5124b91 2024-11-19T00:14:34,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/4de46692345a46638ccd7f9c4de485a1 is 50, key is test_row_0/B:col10/1731975273067/Put/seqid=0 2024-11-19T00:14:34,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T00:14:34,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:34,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:34,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:34,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:34,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:34,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:34,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742118_1294 (size=12301) 2024-11-19T00:14:34,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T00:14:34,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:34,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:34,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:34,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:34,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T00:14:34,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975334883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:34,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:34,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:34,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:34,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975334883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:34,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:34,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:34,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:34,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T00:14:34,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:34,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:34,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:34,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:34,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:35,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/4de46692345a46638ccd7f9c4de485a1 2024-11-19T00:14:35,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/65b8d4472334473b9c10e061d31527f9 is 50, key is test_row_0/C:col10/1731975273067/Put/seqid=0 2024-11-19T00:14:35,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742119_1295 (size=12301) 2024-11-19T00:14:35,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T00:14:35,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:35,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:35,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:35,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:35,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T00:14:35,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T00:14:35,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:35,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:35,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:35,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:35,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:35,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:35,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:35,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975335393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:35,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:35,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975335394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:35,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T00:14:35,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:35,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:35,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:35,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:35,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:35,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/65b8d4472334473b9c10e061d31527f9 2024-11-19T00:14:35,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/b654684f4b0d4df288130463d5124b91 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/b654684f4b0d4df288130463d5124b91 2024-11-19T00:14:35,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/b654684f4b0d4df288130463d5124b91, entries=200, sequenceid=276, filesize=14.4 K 2024-11-19T00:14:35,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/4de46692345a46638ccd7f9c4de485a1 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/4de46692345a46638ccd7f9c4de485a1 2024-11-19T00:14:35,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/4de46692345a46638ccd7f9c4de485a1, entries=150, sequenceid=276, filesize=12.0 K 2024-11-19T00:14:35,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/65b8d4472334473b9c10e061d31527f9 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/65b8d4472334473b9c10e061d31527f9 2024-11-19T00:14:35,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/65b8d4472334473b9c10e061d31527f9, entries=150, sequenceid=276, filesize=12.0 K 2024-11-19T00:14:35,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for c24634f4d3f0bb64baf797330c06cc91 in 1296ms, sequenceid=276, compaction requested=true 2024-11-19T00:14:35,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:35,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:14:35,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:35,523 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:35,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:14:35,523 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:35,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:35,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:14:35,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:35,524 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:35,524 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:35,524 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/A is initiating minor compaction (all files) 2024-11-19T00:14:35,524 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/B is initiating minor compaction (all files) 2024-11-19T00:14:35,524 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/B in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:35,524 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/A in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:35,524 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/dd437c6a14284541b5e9d5f94326f4bd, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/c82c498f6dca49b392a21085aa886b00, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/b654684f4b0d4df288130463d5124b91] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=38.7 K 2024-11-19T00:14:35,524 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/f7dbdf918fa54d76b6ce8bfc0b753614, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/9de960dc8a6d48809bb7361cf7a2fd89, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/4de46692345a46638ccd7f9c4de485a1] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=36.3 K 2024-11-19T00:14:35,525 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting f7dbdf918fa54d76b6ce8bfc0b753614, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731975270666 2024-11-19T00:14:35,525 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd437c6a14284541b5e9d5f94326f4bd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731975270666 2024-11-19T00:14:35,525 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 9de960dc8a6d48809bb7361cf7a2fd89, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731975271828 2024-11-19T00:14:35,525 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting c82c498f6dca49b392a21085aa886b00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731975271828 2024-11-19T00:14:35,525 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 4de46692345a46638ccd7f9c4de485a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731975273067 2024-11-19T00:14:35,525 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting b654684f4b0d4df288130463d5124b91, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731975273067 2024-11-19T00:14:35,536 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#B#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:35,536 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/91497e9fb362484da9bf7f14580d2189 is 50, key is test_row_0/B:col10/1731975273067/Put/seqid=0 2024-11-19T00:14:35,539 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#A#compaction#250 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:35,540 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/42372bb7caeb450ebaa0387c50f540d5 is 50, key is test_row_0/A:col10/1731975273067/Put/seqid=0 2024-11-19T00:14:35,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742120_1296 (size=12949) 2024-11-19T00:14:35,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742121_1297 (size=12949) 2024-11-19T00:14:35,556 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/42372bb7caeb450ebaa0387c50f540d5 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/42372bb7caeb450ebaa0387c50f540d5 2024-11-19T00:14:35,561 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/A of c24634f4d3f0bb64baf797330c06cc91 into 42372bb7caeb450ebaa0387c50f540d5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:35,561 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:35,561 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/A, priority=13, startTime=1731975275522; duration=0sec 2024-11-19T00:14:35,561 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:35,561 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:A 2024-11-19T00:14:35,561 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:35,562 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:35,562 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/C is initiating minor compaction (all files) 2024-11-19T00:14:35,562 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/C in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:35,562 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/6cfc079d09304998b3aaae783145bae5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/f411809aafb04580b87edb43f7d8d253, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/65b8d4472334473b9c10e061d31527f9] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=36.3 K 2024-11-19T00:14:35,563 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cfc079d09304998b3aaae783145bae5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731975270666 2024-11-19T00:14:35,563 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting f411809aafb04580b87edb43f7d8d253, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731975271828 2024-11-19T00:14:35,563 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65b8d4472334473b9c10e061d31527f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731975273067 2024-11-19T00:14:35,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T00:14:35,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:35,568 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing c24634f4d3f0bb64baf797330c06cc91 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T00:14:35,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=A 2024-11-19T00:14:35,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:35,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=B 2024-11-19T00:14:35,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:35,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=C 2024-11-19T00:14:35,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:35,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/b7a7b6e1be8948b5a75b7ba27e6def55 is 50, key is test_row_0/A:col10/1731975274241/Put/seqid=0 2024-11-19T00:14:35,580 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#C#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:35,581 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/80e9f7522c854460a239be04ac05572b is 50, key is test_row_0/C:col10/1731975273067/Put/seqid=0 2024-11-19T00:14:35,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742123_1299 (size=12949) 2024-11-19T00:14:35,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742122_1298 (size=12301) 2024-11-19T00:14:35,607 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/b7a7b6e1be8948b5a75b7ba27e6def55 2024-11-19T00:14:35,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/2e225a50553a447fa64e34c7b9556a5d is 50, key is test_row_0/B:col10/1731975274241/Put/seqid=0 2024-11-19T00:14:35,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742124_1300 (size=12301) 2024-11-19T00:14:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T00:14:35,952 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/91497e9fb362484da9bf7f14580d2189 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/91497e9fb362484da9bf7f14580d2189 2024-11-19T00:14:35,959 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/B of c24634f4d3f0bb64baf797330c06cc91 into 91497e9fb362484da9bf7f14580d2189(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:35,959 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:35,959 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/B, priority=13, startTime=1731975275523; duration=0sec 2024-11-19T00:14:35,959 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:35,960 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:B 2024-11-19T00:14:36,012 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/80e9f7522c854460a239be04ac05572b as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/80e9f7522c854460a239be04ac05572b 2024-11-19T00:14:36,018 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/C of c24634f4d3f0bb64baf797330c06cc91 into 80e9f7522c854460a239be04ac05572b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:36,018 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:36,018 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/C, priority=13, startTime=1731975275523; duration=0sec 2024-11-19T00:14:36,018 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:36,018 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:C 2024-11-19T00:14:36,025 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/2e225a50553a447fa64e34c7b9556a5d 2024-11-19T00:14:36,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/decb60b2d4024e0eaace1ca5324f0bdb is 50, key is test_row_0/C:col10/1731975274241/Put/seqid=0 2024-11-19T00:14:36,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742125_1301 (size=12301) 2024-11-19T00:14:36,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:36,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:36,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975276431, max timestamp=1731975276444Num uniques 859; 2024-11-19T00:14:36,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:14:36,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975276431, max timestamp=1731975276444Num uniques 859; 2024-11-19T00:14:36,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975276431, max timestamp=1731975276444Num uniques 859; 2024-11-19T00:14:36,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:14:36,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:14:36,450 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/decb60b2d4024e0eaace1ca5324f0bdb 2024-11-19T00:14:36,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/b7a7b6e1be8948b5a75b7ba27e6def55 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/b7a7b6e1be8948b5a75b7ba27e6def55 2024-11-19T00:14:36,460 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/b7a7b6e1be8948b5a75b7ba27e6def55, entries=150, sequenceid=287, filesize=12.0 K 2024-11-19T00:14:36,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/2e225a50553a447fa64e34c7b9556a5d as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/2e225a50553a447fa64e34c7b9556a5d 2024-11-19T00:14:36,465 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/2e225a50553a447fa64e34c7b9556a5d, entries=150, sequenceid=287, filesize=12.0 K 2024-11-19T00:14:36,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/decb60b2d4024e0eaace1ca5324f0bdb as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/decb60b2d4024e0eaace1ca5324f0bdb 2024-11-19T00:14:36,471 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/decb60b2d4024e0eaace1ca5324f0bdb, entries=150, sequenceid=287, filesize=12.0 K 2024-11-19T00:14:36,472 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=73.80 KB/75570 for c24634f4d3f0bb64baf797330c06cc91 in 904ms, sequenceid=287, compaction requested=false 2024-11-19T00:14:36,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:36,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:36,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-19T00:14:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-19T00:14:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:36,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c24634f4d3f0bb64baf797330c06cc91 3/3 column families, dataSize=87.22 KB heapSize=240.84 KB 2024-11-19T00:14:36,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=A 2024-11-19T00:14:36,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:14:36,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=B 2024-11-19T00:14:36,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:14:36,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=C 2024-11-19T00:14:36,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:14:36,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-19T00:14:36,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9780 sec 2024-11-19T00:14:36,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.9830 sec 2024-11-19T00:14:36,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/95ace14d39e64dcea59e4ccf8ba43f08 is 50, key is test_row_0/A:col10/1731975276474/Put/seqid=0 2024-11-19T00:14:36,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742126_1302 (size=17181) 2024-11-19T00:14:36,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975336523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:36,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:36,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:36,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:36,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975336524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:36,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:36,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:36,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:36,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975336634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:36,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975336634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:36,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:36,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:36,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:36,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:36,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:36,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T00:14:36,644 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:14:36,644 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:14:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-19T00:14:36,646 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:14:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T00:14:36,646 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:14:36,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:14:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T00:14:36,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T00:14:36,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:36,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:36,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:36,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:36,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:36,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975336844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:36,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:36,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975336844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:36,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:36,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:36,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:36,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:36,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:36,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/95ace14d39e64dcea59e4ccf8ba43f08 2024-11-19T00:14:36,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/6e4f09a89fc44dce9163d5e23481e552 is 50, key is test_row_0/B:col10/1731975276474/Put/seqid=0 2024-11-19T00:14:36,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742127_1303 (size=12301) 2024-11-19T00:14:36,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T00:14:36,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:36,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:36,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:36,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:36,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:36,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:36,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T00:14:37,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T00:14:37,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:37,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:37,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:37,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:37,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:37,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975337154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:37,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975337154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:37,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T00:14:37,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:37,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:37,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:37,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:37,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T00:14:37,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/6e4f09a89fc44dce9163d5e23481e552 2024-11-19T00:14:37,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/da9e5058369f4a63a79d967f0cb6371e is 50, key is test_row_0/C:col10/1731975276474/Put/seqid=0 2024-11-19T00:14:37,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742128_1304 (size=12301) 2024-11-19T00:14:37,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/da9e5058369f4a63a79d967f0cb6371e 2024-11-19T00:14:37,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/95ace14d39e64dcea59e4ccf8ba43f08 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/95ace14d39e64dcea59e4ccf8ba43f08 2024-11-19T00:14:37,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/95ace14d39e64dcea59e4ccf8ba43f08, entries=250, sequenceid=318, filesize=16.8 K 2024-11-19T00:14:37,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/6e4f09a89fc44dce9163d5e23481e552 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/6e4f09a89fc44dce9163d5e23481e552 2024-11-19T00:14:37,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/6e4f09a89fc44dce9163d5e23481e552, entries=150, sequenceid=318, filesize=12.0 K 2024-11-19T00:14:37,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/da9e5058369f4a63a79d967f0cb6371e as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/da9e5058369f4a63a79d967f0cb6371e 2024-11-19T00:14:37,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/da9e5058369f4a63a79d967f0cb6371e, entries=150, sequenceid=318, filesize=12.0 K 2024-11-19T00:14:37,348 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-19T00:14:37,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~240.80 KB/246576, currentSize=100.63 KB/103050 for c24634f4d3f0bb64baf797330c06cc91 in 873ms, sequenceid=318, compaction requested=true 2024-11-19T00:14:37,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:37,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:14:37,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:37,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:14:37,350 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:37,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:37,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:14:37,350 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:37,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:37,351 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:37,351 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/B is initiating minor compaction (all files) 2024-11-19T00:14:37,351 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42431 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:37,351 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/A is initiating minor compaction (all files) 2024-11-19T00:14:37,351 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/B in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:37,351 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/A in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:37,351 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/42372bb7caeb450ebaa0387c50f540d5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/b7a7b6e1be8948b5a75b7ba27e6def55, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/95ace14d39e64dcea59e4ccf8ba43f08] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=41.4 K 2024-11-19T00:14:37,351 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/91497e9fb362484da9bf7f14580d2189, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/2e225a50553a447fa64e34c7b9556a5d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/6e4f09a89fc44dce9163d5e23481e552] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=36.7 K 2024-11-19T00:14:37,353 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 91497e9fb362484da9bf7f14580d2189, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731975273067 2024-11-19T00:14:37,353 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42372bb7caeb450ebaa0387c50f540d5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731975273067 2024-11-19T00:14:37,353 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7a7b6e1be8948b5a75b7ba27e6def55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731975274241 2024-11-19T00:14:37,353 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e225a50553a447fa64e34c7b9556a5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731975274241 2024-11-19T00:14:37,354 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e4f09a89fc44dce9163d5e23481e552, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731975276450 2024-11-19T00:14:37,354 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95ace14d39e64dcea59e4ccf8ba43f08, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731975276450 2024-11-19T00:14:37,362 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#A#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:37,362 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/c523adc21f1d41d88711869c6dc15497 is 50, key is test_row_0/A:col10/1731975276474/Put/seqid=0 2024-11-19T00:14:37,363 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#B#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:37,363 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/7fb91553901f423887d5f96c36b05fad is 50, key is test_row_0/B:col10/1731975276474/Put/seqid=0 2024-11-19T00:14:37,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742129_1305 (size=13051) 2024-11-19T00:14:37,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742130_1306 (size=13051) 2024-11-19T00:14:37,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T00:14:37,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:37,413 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing c24634f4d3f0bb64baf797330c06cc91 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T00:14:37,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=A 2024-11-19T00:14:37,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:37,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=B 2024-11-19T00:14:37,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:37,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=C 2024-11-19T00:14:37,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:37,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/098fa6991be54505b4e63959ce359ba9 is 50, key is test_row_0/A:col10/1731975276514/Put/seqid=0 2024-11-19T00:14:37,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742131_1307 (size=12301) 2024-11-19T00:14:37,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:37,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:37,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:37,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975337687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:37,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:37,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:37,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:37,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:37,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975337691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:37,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:37,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:37,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:37,781 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/c523adc21f1d41d88711869c6dc15497 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/c523adc21f1d41d88711869c6dc15497 2024-11-19T00:14:37,781 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/7fb91553901f423887d5f96c36b05fad as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/7fb91553901f423887d5f96c36b05fad 2024-11-19T00:14:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T00:14:37,786 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/A of c24634f4d3f0bb64baf797330c06cc91 into c523adc21f1d41d88711869c6dc15497(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:37,786 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/B of c24634f4d3f0bb64baf797330c06cc91 into 7fb91553901f423887d5f96c36b05fad(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:37,787 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:37,787 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:37,787 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/B, priority=13, startTime=1731975277350; duration=0sec 2024-11-19T00:14:37,787 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/A, priority=13, startTime=1731975277350; duration=0sec 2024-11-19T00:14:37,787 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:37,787 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:A 2024-11-19T00:14:37,787 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:37,787 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:B 2024-11-19T00:14:37,787 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:37,788 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:37,788 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/C is initiating minor compaction (all files) 2024-11-19T00:14:37,789 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/C in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:37,789 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/80e9f7522c854460a239be04ac05572b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/decb60b2d4024e0eaace1ca5324f0bdb, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/da9e5058369f4a63a79d967f0cb6371e] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=36.7 K 2024-11-19T00:14:37,789 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80e9f7522c854460a239be04ac05572b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731975273067 2024-11-19T00:14:37,789 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting decb60b2d4024e0eaace1ca5324f0bdb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731975274241 2024-11-19T00:14:37,790 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting da9e5058369f4a63a79d967f0cb6371e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731975276450 2024-11-19T00:14:37,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:37,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975337793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:37,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:37,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:37,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:37,798 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#C#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:37,798 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/546c8d7569f54cdf92a405f5890f9564 is 50, key is test_row_0/C:col10/1731975276474/Put/seqid=0 2024-11-19T00:14:37,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975337803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:37,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:37,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:37,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:37,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742132_1308 (size=13051) 2024-11-19T00:14:37,817 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/546c8d7569f54cdf92a405f5890f9564 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/546c8d7569f54cdf92a405f5890f9564 2024-11-19T00:14:37,824 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/098fa6991be54505b4e63959ce359ba9 2024-11-19T00:14:37,826 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/C of c24634f4d3f0bb64baf797330c06cc91 into 546c8d7569f54cdf92a405f5890f9564(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:37,826 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:37,826 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/C, priority=13, startTime=1731975277350; duration=0sec 2024-11-19T00:14:37,826 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:37,826 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:C 2024-11-19T00:14:37,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/4a5b2f6969cf4f65a2974a6e12fe0a1d is 50, key is test_row_0/B:col10/1731975276514/Put/seqid=0 2024-11-19T00:14:37,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742133_1309 (size=12301) 2024-11-19T00:14:38,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975338003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:38,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:38,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:38,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:38,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975338013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:38,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:38,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:38,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:38,241 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/4a5b2f6969cf4f65a2974a6e12fe0a1d 2024-11-19T00:14:38,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/4eb06bd2f2194bd08d973ead1b74568a is 50, key is test_row_0/C:col10/1731975276514/Put/seqid=0 2024-11-19T00:14:38,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742134_1310 (size=12301) 2024-11-19T00:14:38,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975338314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:38,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:38,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:38,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:38,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975338324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:38,658 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/4eb06bd2f2194bd08d973ead1b74568a 2024-11-19T00:14:38,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/098fa6991be54505b4e63959ce359ba9 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/098fa6991be54505b4e63959ce359ba9 2024-11-19T00:14:38,673 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/098fa6991be54505b4e63959ce359ba9, entries=150, sequenceid=336, filesize=12.0 K 2024-11-19T00:14:38,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/4a5b2f6969cf4f65a2974a6e12fe0a1d as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/4a5b2f6969cf4f65a2974a6e12fe0a1d 2024-11-19T00:14:38,678 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/4a5b2f6969cf4f65a2974a6e12fe0a1d, entries=150, sequenceid=336, filesize=12.0 K 2024-11-19T00:14:38,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/4eb06bd2f2194bd08d973ead1b74568a as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/4eb06bd2f2194bd08d973ead1b74568a 2024-11-19T00:14:38,683 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/4eb06bd2f2194bd08d973ead1b74568a, entries=150, sequenceid=336, filesize=12.0 K 2024-11-19T00:14:38,684 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c24634f4d3f0bb64baf797330c06cc91 in 1271ms, sequenceid=336, compaction requested=false 2024-11-19T00:14:38,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:38,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:38,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-19T00:14:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-19T00:14:38,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-19T00:14:38,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0380 sec 2024-11-19T00:14:38,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.0440 sec 2024-11-19T00:14:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T00:14:38,794 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:14:38,794 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:14:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-19T00:14:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T00:14:38,796 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:14:38,797 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:14:38,797 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:14:38,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:38,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c24634f4d3f0bb64baf797330c06cc91 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T00:14:38,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=A 2024-11-19T00:14:38,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:38,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=B 2024-11-19T00:14:38,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:38,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=C 2024-11-19T00:14:38,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:38,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/863f1ee0c11d481e9e904b657ef6ced9 is 50, key is test_row_0/A:col10/1731975278824/Put/seqid=0 2024-11-19T00:14:38,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742135_1311 (size=12301) 2024-11-19T00:14:38,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/863f1ee0c11d481e9e904b657ef6ced9 2024-11-19T00:14:38,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/9af60292266149e287e88b036438bc74 is 50, key is test_row_0/B:col10/1731975278824/Put/seqid=0 2024-11-19T00:14:38,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742136_1312 (size=12301) 2024-11-19T00:14:38,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:38,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975338870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:38,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:38,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:38,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:38,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:38,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975338877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:38,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:38,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:38,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:38,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T00:14:38,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T00:14:38,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:38,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:38,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:38,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:38,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:38,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:38,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:38,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975338984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:38,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:38,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:38,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:38,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:38,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975338984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:39,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T00:14:39,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:39,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:39,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:39,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:39,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T00:14:39,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975339194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:39,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:39,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:39,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:39,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975339194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:39,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:39,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:39,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:39,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T00:14:39,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:39,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:39,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:39,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:39,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:39,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/9af60292266149e287e88b036438bc74 2024-11-19T00:14:39,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/d5558fb544ce4f2789cd302dcf71019e is 50, key is test_row_0/C:col10/1731975278824/Put/seqid=0 2024-11-19T00:14:39,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742137_1313 (size=12301) 2024-11-19T00:14:39,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T00:14:39,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:39,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:39,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:39,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:39,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:39,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T00:14:39,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:39,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975339504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:39,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:39,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:39,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975339504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:39,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:39,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:39,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:39,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:39,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:39,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T00:14:39,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:39,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:39,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:39,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:39,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:39,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/d5558fb544ce4f2789cd302dcf71019e 2024-11-19T00:14:39,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/863f1ee0c11d481e9e904b657ef6ced9 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/863f1ee0c11d481e9e904b657ef6ced9 2024-11-19T00:14:39,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/863f1ee0c11d481e9e904b657ef6ced9, entries=150, sequenceid=354, filesize=12.0 K 2024-11-19T00:14:39,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/9af60292266149e287e88b036438bc74 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/9af60292266149e287e88b036438bc74 2024-11-19T00:14:39,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/9af60292266149e287e88b036438bc74, entries=150, sequenceid=354, filesize=12.0 K 2024-11-19T00:14:39,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/d5558fb544ce4f2789cd302dcf71019e as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/d5558fb544ce4f2789cd302dcf71019e 2024-11-19T00:14:39,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/d5558fb544ce4f2789cd302dcf71019e, entries=150, sequenceid=354, filesize=12.0 K 2024-11-19T00:14:39,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c24634f4d3f0bb64baf797330c06cc91 in 878ms, sequenceid=354, compaction requested=true 2024-11-19T00:14:39,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:39,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:14:39,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:39,703 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:39,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:14:39,704 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:39,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:39,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:14:39,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:39,704 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:39,704 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:39,704 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/A is initiating minor compaction (all files) 2024-11-19T00:14:39,704 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/B is initiating minor compaction (all files) 2024-11-19T00:14:39,704 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/A in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:39,704 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/B in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:39,704 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/c523adc21f1d41d88711869c6dc15497, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/098fa6991be54505b4e63959ce359ba9, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/863f1ee0c11d481e9e904b657ef6ced9] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=36.8 K 2024-11-19T00:14:39,704 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/7fb91553901f423887d5f96c36b05fad, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/4a5b2f6969cf4f65a2974a6e12fe0a1d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/9af60292266149e287e88b036438bc74] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=36.8 K 2024-11-19T00:14:39,705 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fb91553901f423887d5f96c36b05fad, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731975276450 2024-11-19T00:14:39,705 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting c523adc21f1d41d88711869c6dc15497, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731975276450 2024-11-19T00:14:39,706 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a5b2f6969cf4f65a2974a6e12fe0a1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731975276514 2024-11-19T00:14:39,706 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 098fa6991be54505b4e63959ce359ba9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731975276514 2024-11-19T00:14:39,706 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 9af60292266149e287e88b036438bc74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731975277678 2024-11-19T00:14:39,706 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 863f1ee0c11d481e9e904b657ef6ced9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731975277678 2024-11-19T00:14:39,712 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#B#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:39,713 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/1802c3a22d5147df9211791c850e5431 is 50, key is test_row_0/B:col10/1731975278824/Put/seqid=0 2024-11-19T00:14:39,715 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#A#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:39,715 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/bbce5b5b771b4a378f0960d07b6a1d5f is 50, key is test_row_0/A:col10/1731975278824/Put/seqid=0 2024-11-19T00:14:39,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T00:14:39,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:39,720 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing c24634f4d3f0bb64baf797330c06cc91 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T00:14:39,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=A 2024-11-19T00:14:39,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:39,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=B 2024-11-19T00:14:39,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:39,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=C 2024-11-19T00:14:39,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:39,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742139_1315 (size=13153) 2024-11-19T00:14:39,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/15e4f38a6d3b4c0fbcfe5b923763e24e is 50, key is test_row_0/A:col10/1731975278869/Put/seqid=0 2024-11-19T00:14:39,748 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/1802c3a22d5147df9211791c850e5431 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/1802c3a22d5147df9211791c850e5431 2024-11-19T00:14:39,753 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/B of c24634f4d3f0bb64baf797330c06cc91 into 1802c3a22d5147df9211791c850e5431(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:39,753 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:39,753 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/B, priority=13, startTime=1731975279703; duration=0sec 2024-11-19T00:14:39,753 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:39,753 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:B 2024-11-19T00:14:39,753 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:39,754 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:39,754 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/C is initiating minor compaction (all files) 2024-11-19T00:14:39,754 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/C in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:39,754 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/546c8d7569f54cdf92a405f5890f9564, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/4eb06bd2f2194bd08d973ead1b74568a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/d5558fb544ce4f2789cd302dcf71019e] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=36.8 K 2024-11-19T00:14:39,755 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 546c8d7569f54cdf92a405f5890f9564, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731975276450 2024-11-19T00:14:39,756 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 4eb06bd2f2194bd08d973ead1b74568a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731975276514 2024-11-19T00:14:39,756 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting d5558fb544ce4f2789cd302dcf71019e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731975277678 2024-11-19T00:14:39,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742138_1314 (size=13153) 2024-11-19T00:14:39,769 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/bbce5b5b771b4a378f0960d07b6a1d5f as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/bbce5b5b771b4a378f0960d07b6a1d5f 2024-11-19T00:14:39,772 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#C#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:39,773 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/cd2628dcecd349559580b9f7ae38dbca is 50, key is test_row_0/C:col10/1731975278824/Put/seqid=0 2024-11-19T00:14:39,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742140_1316 (size=12301) 2024-11-19T00:14:39,776 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/15e4f38a6d3b4c0fbcfe5b923763e24e 2024-11-19T00:14:39,781 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/A of c24634f4d3f0bb64baf797330c06cc91 into bbce5b5b771b4a378f0960d07b6a1d5f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:39,782 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:39,782 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/A, priority=13, startTime=1731975279703; duration=0sec 2024-11-19T00:14:39,782 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:39,782 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:A 2024-11-19T00:14:39,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/d777b4ad7276461eb51eea9f1a779685 is 50, key is test_row_0/B:col10/1731975278869/Put/seqid=0 2024-11-19T00:14:39,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742141_1317 (size=13153) 2024-11-19T00:14:39,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742142_1318 (size=12301) 2024-11-19T00:14:39,795 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/d777b4ad7276461eb51eea9f1a779685 2024-11-19T00:14:39,801 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/cd2628dcecd349559580b9f7ae38dbca as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/cd2628dcecd349559580b9f7ae38dbca 2024-11-19T00:14:39,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/eb87fc7b572043e8bb561078372b61ca is 50, key is test_row_0/C:col10/1731975278869/Put/seqid=0 2024-11-19T00:14:39,807 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/C of c24634f4d3f0bb64baf797330c06cc91 into cd2628dcecd349559580b9f7ae38dbca(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:39,807 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:39,809 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/C, priority=13, startTime=1731975279704; duration=0sec 2024-11-19T00:14:39,809 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:39,809 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:C 2024-11-19T00:14:39,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742143_1319 (size=12301) 2024-11-19T00:14:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T00:14:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:40,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:40,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975340040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:40,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:40,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975340041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:40,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:40,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975340154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:40,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:40,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975340154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:40,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:40,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:40,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51968 deadline: 1731975340163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:40,166 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_1' in c24634f4d3f0bb64baf797330c06cc91 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:14:40,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:40,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1731975340204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:40,207 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_1' in c24634f4d3f0bb64baf797330c06cc91 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18284 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:14:40,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:40,215 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/eb87fc7b572043e8bb561078372b61ca 2024-11-19T00:14:40,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/15e4f38a6d3b4c0fbcfe5b923763e24e as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/15e4f38a6d3b4c0fbcfe5b923763e24e 2024-11-19T00:14:40,223 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/15e4f38a6d3b4c0fbcfe5b923763e24e, entries=150, sequenceid=371, filesize=12.0 K 2024-11-19T00:14:40,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/d777b4ad7276461eb51eea9f1a779685 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/d777b4ad7276461eb51eea9f1a779685 2024-11-19T00:14:40,228 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/d777b4ad7276461eb51eea9f1a779685, entries=150, sequenceid=371, filesize=12.0 K 2024-11-19T00:14:40,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/eb87fc7b572043e8bb561078372b61ca as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/eb87fc7b572043e8bb561078372b61ca 2024-11-19T00:14:40,233 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/eb87fc7b572043e8bb561078372b61ca, entries=150, sequenceid=371, filesize=12.0 K 2024-11-19T00:14:40,235 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c24634f4d3f0bb64baf797330c06cc91 in 516ms, sequenceid=371, compaction requested=false 2024-11-19T00:14:40,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:40,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:40,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-19T00:14:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-19T00:14:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:40,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c24634f4d3f0bb64baf797330c06cc91 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T00:14:40,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=A 2024-11-19T00:14:40,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:40,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=B 2024-11-19T00:14:40,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:40,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=C 2024-11-19T00:14:40,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:40,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-19T00:14:40,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4390 sec 2024-11-19T00:14:40,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.4450 sec 2024-11-19T00:14:40,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/e0d09775920146f0a640cb4e096b5088 is 50, key is test_row_0/A:col10/1731975280037/Put/seqid=0 2024-11-19T00:14:40,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742144_1320 (size=14741) 2024-11-19T00:14:40,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51992 deadline: 1731975340293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:40,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:40,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975340364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:40,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975340364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:40,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:40,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51992 deadline: 1731975340403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:40,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:40,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:40,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51992 deadline: 1731975340614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:40,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:40,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/e0d09775920146f0a640cb4e096b5088 2024-11-19T00:14:40,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/9f8b6dd6134d4d928d2425815203530d is 50, key is test_row_0/B:col10/1731975280037/Put/seqid=0 2024-11-19T00:14:40,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742145_1321 (size=12301) 2024-11-19T00:14:40,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/9f8b6dd6134d4d928d2425815203530d 2024-11-19T00:14:40,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:40,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975340674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:40,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975340674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:40,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/4b22e7a123134d758121709e8b1a2d31 is 50, key is test_row_0/C:col10/1731975280037/Put/seqid=0 2024-11-19T00:14:40,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742146_1322 (size=12301) 2024-11-19T00:14:40,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/4b22e7a123134d758121709e8b1a2d31 2024-11-19T00:14:40,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/e0d09775920146f0a640cb4e096b5088 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/e0d09775920146f0a640cb4e096b5088 2024-11-19T00:14:40,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/e0d09775920146f0a640cb4e096b5088, entries=200, sequenceid=390, filesize=14.4 K 2024-11-19T00:14:40,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/9f8b6dd6134d4d928d2425815203530d as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/9f8b6dd6134d4d928d2425815203530d 2024-11-19T00:14:40,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/9f8b6dd6134d4d928d2425815203530d, entries=150, sequenceid=390, filesize=12.0 K 2024-11-19T00:14:40,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/4b22e7a123134d758121709e8b1a2d31 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/4b22e7a123134d758121709e8b1a2d31 2024-11-19T00:14:40,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/4b22e7a123134d758121709e8b1a2d31, entries=150, sequenceid=390, filesize=12.0 K 2024-11-19T00:14:40,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c24634f4d3f0bb64baf797330c06cc91 in 498ms, sequenceid=390, compaction requested=true 2024-11-19T00:14:40,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:40,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:14:40,735 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:40,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:40,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:14:40,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:40,736 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:40,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c24634f4d3f0bb64baf797330c06cc91:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:14:40,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:40,737 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:40,737 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/A is initiating minor compaction (all files) 2024-11-19T00:14:40,737 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/A in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:40,737 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/bbce5b5b771b4a378f0960d07b6a1d5f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/15e4f38a6d3b4c0fbcfe5b923763e24e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/e0d09775920146f0a640cb4e096b5088] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=39.3 K 2024-11-19T00:14:40,738 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbce5b5b771b4a378f0960d07b6a1d5f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731975277678 2024-11-19T00:14:40,738 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:40,738 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/B is initiating minor compaction (all files) 2024-11-19T00:14:40,738 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/B in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:40,738 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15e4f38a6d3b4c0fbcfe5b923763e24e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731975278869 2024-11-19T00:14:40,738 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/1802c3a22d5147df9211791c850e5431, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/d777b4ad7276461eb51eea9f1a779685, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/9f8b6dd6134d4d928d2425815203530d] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=36.9 K 2024-11-19T00:14:40,739 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 1802c3a22d5147df9211791c850e5431, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731975277678 2024-11-19T00:14:40,739 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0d09775920146f0a640cb4e096b5088, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731975280032 2024-11-19T00:14:40,739 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting d777b4ad7276461eb51eea9f1a779685, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731975278869 2024-11-19T00:14:40,740 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f8b6dd6134d4d928d2425815203530d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731975280037 2024-11-19T00:14:40,750 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#B#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:40,750 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/b982449a36444ea4a2a7901869b1195b is 50, key is test_row_0/B:col10/1731975280037/Put/seqid=0 2024-11-19T00:14:40,757 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#A#compaction#277 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:40,757 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/b6f84613e39e49cbbedb93fcff03a95e is 50, key is test_row_0/A:col10/1731975280037/Put/seqid=0 2024-11-19T00:14:40,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742147_1323 (size=13255) 2024-11-19T00:14:40,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742148_1324 (size=13255) 2024-11-19T00:14:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:40,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c24634f4d3f0bb64baf797330c06cc91 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T00:14:40,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=A 2024-11-19T00:14:40,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:40,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=B 2024-11-19T00:14:40,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:40,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=C 2024-11-19T00:14:40,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:40,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/33b43e1ea7fd493ba5f4ddc1d4d992fc is 50, key is test_row_0/A:col10/1731975280292/Put/seqid=0 2024-11-19T00:14:40,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742149_1325 (size=9857) 2024-11-19T00:14:40,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/33b43e1ea7fd493ba5f4ddc1d4d992fc 2024-11-19T00:14:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T00:14:40,944 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:14:40,944 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:14:40,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-19T00:14:40,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T00:14:40,947 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:14:40,948 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:14:40,948 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:14:40,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/f7e00fc92ee7497eb3765bff1ff009a8 is 50, key is test_row_0/B:col10/1731975280292/Put/seqid=0 2024-11-19T00:14:40,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742150_1326 (size=9857) 2024-11-19T00:14:40,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:40,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51992 deadline: 1731975340985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:41,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T00:14:41,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:41,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51992 deadline: 1731975341094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:41,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T00:14:41,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:41,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:41,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:41,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:41,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:41,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:41,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:41,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1731975341184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:41,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:41,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:41,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51966 deadline: 1731975341184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:41,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:41,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:41,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:41,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:41,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:41,192 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/b982449a36444ea4a2a7901869b1195b as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/b982449a36444ea4a2a7901869b1195b 2024-11-19T00:14:41,197 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/B of c24634f4d3f0bb64baf797330c06cc91 into b982449a36444ea4a2a7901869b1195b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:41,197 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:41,197 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/B, priority=13, startTime=1731975280736; duration=0sec 2024-11-19T00:14:41,197 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:41,197 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:B 2024-11-19T00:14:41,197 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:41,198 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:41,198 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): c24634f4d3f0bb64baf797330c06cc91/C is initiating minor compaction (all files) 2024-11-19T00:14:41,198 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c24634f4d3f0bb64baf797330c06cc91/C in TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:41,198 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/cd2628dcecd349559580b9f7ae38dbca, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/eb87fc7b572043e8bb561078372b61ca, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/4b22e7a123134d758121709e8b1a2d31] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp, totalSize=36.9 K 2024-11-19T00:14:41,199 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting cd2628dcecd349559580b9f7ae38dbca, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731975277678 2024-11-19T00:14:41,199 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting eb87fc7b572043e8bb561078372b61ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731975278869 2024-11-19T00:14:41,200 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b22e7a123134d758121709e8b1a2d31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731975280037 2024-11-19T00:14:41,204 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/b6f84613e39e49cbbedb93fcff03a95e as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/b6f84613e39e49cbbedb93fcff03a95e 2024-11-19T00:14:41,209 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c24634f4d3f0bb64baf797330c06cc91#C#compaction#280 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:41,210 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/d0bd11ccff404533a7ca963a02a72b91 is 50, key is test_row_0/C:col10/1731975280037/Put/seqid=0 2024-11-19T00:14:41,212 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/A of c24634f4d3f0bb64baf797330c06cc91 into b6f84613e39e49cbbedb93fcff03a95e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:41,212 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:41,212 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/A, priority=13, startTime=1731975280735; duration=0sec 2024-11-19T00:14:41,212 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:41,212 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:A 2024-11-19T00:14:41,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742151_1327 (size=13255) 2024-11-19T00:14:41,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T00:14:41,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:41,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:41,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:41,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:41,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:41,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:41,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T00:14:41,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:41,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51992 deadline: 1731975341303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:41,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/f7e00fc92ee7497eb3765bff1ff009a8 2024-11-19T00:14:41,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/f944ba86ea22493890b7652873f014de is 50, key is test_row_0/C:col10/1731975280292/Put/seqid=0 2024-11-19T00:14:41,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742152_1328 (size=9857) 2024-11-19T00:14:41,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T00:14:41,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:41,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:41,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:41,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:41,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:41,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:41,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T00:14:41,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:41,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:41,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:41,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:41,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:41,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T00:14:41,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:41,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51992 deadline: 1731975341614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:41,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:41,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:41,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:14:41,642 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/d0bd11ccff404533a7ca963a02a72b91 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/d0bd11ccff404533a7ca963a02a72b91 2024-11-19T00:14:41,648 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c24634f4d3f0bb64baf797330c06cc91/C of c24634f4d3f0bb64baf797330c06cc91 into d0bd11ccff404533a7ca963a02a72b91(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:41,648 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:41,648 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., storeName=c24634f4d3f0bb64baf797330c06cc91/C, priority=13, startTime=1731975280736; duration=0sec 2024-11-19T00:14:41,648 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:41,648 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c24634f4d3f0bb64baf797330c06cc91:C 2024-11-19T00:14:41,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T00:14:41,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:41,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:41,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:41,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:41,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:41,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:41,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/f944ba86ea22493890b7652873f014de 2024-11-19T00:14:41,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/33b43e1ea7fd493ba5f4ddc1d4d992fc as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/33b43e1ea7fd493ba5f4ddc1d4d992fc 2024-11-19T00:14:41,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/33b43e1ea7fd493ba5f4ddc1d4d992fc, entries=100, sequenceid=407, filesize=9.6 K 2024-11-19T00:14:41,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/f7e00fc92ee7497eb3765bff1ff009a8 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/f7e00fc92ee7497eb3765bff1ff009a8 2024-11-19T00:14:41,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/f7e00fc92ee7497eb3765bff1ff009a8, entries=100, sequenceid=407, filesize=9.6 K 2024-11-19T00:14:41,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/f944ba86ea22493890b7652873f014de as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/f944ba86ea22493890b7652873f014de 2024-11-19T00:14:41,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/f944ba86ea22493890b7652873f014de, entries=100, sequenceid=407, filesize=9.6 K 2024-11-19T00:14:41,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c24634f4d3f0bb64baf797330c06cc91 in 876ms, sequenceid=407, compaction requested=false 2024-11-19T00:14:41,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:41,833 INFO [Thread-1122 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1122. 2024-11-19T00:14:41,833 DEBUG [Thread-1122 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:14:41,833 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:41,833 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:41,833 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:14:41,834 INFO [Thread-1130 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1130. 2024-11-19T00:14:41,834 DEBUG [Thread-1130 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:14:41,834 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:41,834 INFO [Thread-1128 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1128. 2024-11-19T00:14:41,834 DEBUG [Thread-1128 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:14:41,834 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:41,835 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:41,835 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:14:41,835 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:41,835 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:14:41,836 INFO [Thread-1124 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1124. 2024-11-19T00:14:41,836 DEBUG [Thread-1124 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:14:41,836 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:41,836 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:41,836 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:14:41,838 INFO [Thread-1126 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1126. 2024-11-19T00:14:41,838 DEBUG [Thread-1126 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:14:41,838 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:41,838 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:41,838 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:14:41,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T00:14:41,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:41,868 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing c24634f4d3f0bb64baf797330c06cc91 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T00:14:41,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=A 2024-11-19T00:14:41,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:41,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=B 2024-11-19T00:14:41,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:41,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=C 2024-11-19T00:14:41,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:41,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/018cf872516e48d2ac57a743907fdab6 is 50, key is test_row_0/A:col10/1731975280959/Put/seqid=0 2024-11-19T00:14:41,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742153_1329 (size=12301) 2024-11-19T00:14:42,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T00:14:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:42,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. as already flushing 2024-11-19T00:14:42,126 INFO [Thread-1111 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1111. 2024-11-19T00:14:42,126 DEBUG [Thread-1111 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:14:42,126 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:42,126 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:42,126 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:14:42,196 INFO [Thread-1115 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1115. 2024-11-19T00:14:42,196 DEBUG [Thread-1115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:14:42,196 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:42,196 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:42,196 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:14:42,205 INFO [Thread-1113 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1113. 2024-11-19T00:14:42,205 DEBUG [Thread-1113 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:14:42,205 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:42,205 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:42,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:14:42,276 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/018cf872516e48d2ac57a743907fdab6 2024-11-19T00:14:42,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/3da476f3b70848539e0996c6aeaf01ef is 50, key is test_row_0/B:col10/1731975280959/Put/seqid=0 2024-11-19T00:14:42,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742154_1330 (size=12301) 2024-11-19T00:14:42,652 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T00:14:42,690 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/3da476f3b70848539e0996c6aeaf01ef 2024-11-19T00:14:42,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/0a0db085677245d78f9f67d93022ea80 is 50, key is test_row_0/C:col10/1731975280959/Put/seqid=0 2024-11-19T00:14:42,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742155_1331 (size=12301) 2024-11-19T00:14:43,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T00:14:43,104 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/0a0db085677245d78f9f67d93022ea80 2024-11-19T00:14:43,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/018cf872516e48d2ac57a743907fdab6 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/018cf872516e48d2ac57a743907fdab6 2024-11-19T00:14:43,113 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/018cf872516e48d2ac57a743907fdab6, entries=150, sequenceid=425, filesize=12.0 K 2024-11-19T00:14:43,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/3da476f3b70848539e0996c6aeaf01ef as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/3da476f3b70848539e0996c6aeaf01ef 2024-11-19T00:14:43,118 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/3da476f3b70848539e0996c6aeaf01ef, entries=150, sequenceid=425, filesize=12.0 K 2024-11-19T00:14:43,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/0a0db085677245d78f9f67d93022ea80 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/0a0db085677245d78f9f67d93022ea80 2024-11-19T00:14:43,122 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/0a0db085677245d78f9f67d93022ea80, entries=150, sequenceid=425, filesize=12.0 K 2024-11-19T00:14:43,123 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=20.13 KB/20610 for c24634f4d3f0bb64baf797330c06cc91 in 1255ms, sequenceid=425, compaction requested=true 2024-11-19T00:14:43,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for c24634f4d3f0bb64baf797330c06cc91: 2024-11-19T00:14:43,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:43,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-19T00:14:43,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-19T00:14:43,126 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-19T00:14:43,126 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1760 sec 2024-11-19T00:14:43,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.1830 sec 2024-11-19T00:14:45,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T00:14:45,104 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:14:50,227 INFO [Thread-1117 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1117. 2024-11-19T00:14:50,227 DEBUG [Thread-1117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:14:50,227 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:50,227 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:50,227 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:14:50,266 INFO [Thread-1119 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1119. 2024-11-19T00:14:50,266 DEBUG [Thread-1119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:14:50,266 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:50,266 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:50,266 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:14:50,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T00:14:50,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-19T00:14:50,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 137 2024-11-19T00:14:50,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 138 2024-11-19T00:14:50,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-19T00:14:50,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-19T00:14:50,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T00:14:50,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3144 2024-11-19T00:14:50,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3017 2024-11-19T00:14:50,266 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2855 2024-11-19T00:14:50,267 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3133 2024-11-19T00:14:50,267 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3008 2024-11-19T00:14:50,267 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T00:14:50,267 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T00:14:50,267 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T00:14:50,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:50,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:50,267 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:14:50,268 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T00:14:50,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T00:14:50,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T00:14:50,272 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975290271"}]},"ts":"1731975290271"} 2024-11-19T00:14:50,273 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T00:14:50,273 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T00:14:50,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T00:14:50,276 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c24634f4d3f0bb64baf797330c06cc91, UNASSIGN}] 2024-11-19T00:14:50,277 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c24634f4d3f0bb64baf797330c06cc91, UNASSIGN 2024-11-19T00:14:50,278 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=c24634f4d3f0bb64baf797330c06cc91, regionState=CLOSING, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:14:50,279 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c24634f4d3f0bb64baf797330c06cc91, UNASSIGN because future has completed 2024-11-19T00:14:50,280 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T00:14:50,280 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317}] 2024-11-19T00:14:50,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T00:14:50,433 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:50,433 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T00:14:50,433 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing c24634f4d3f0bb64baf797330c06cc91, disabling compactions & flushes 2024-11-19T00:14:50,433 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:50,433 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:50,433 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. after waiting 0 ms 2024-11-19T00:14:50,433 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:50,433 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing c24634f4d3f0bb64baf797330c06cc91 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T00:14:50,433 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=A 2024-11-19T00:14:50,433 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:50,434 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=B 2024-11-19T00:14:50,434 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:50,434 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c24634f4d3f0bb64baf797330c06cc91, store=C 2024-11-19T00:14:50,434 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:50,437 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/8437a2db52374c96bd1b6ce7b003156e is 50, key is test_row_0/A:col10/1731975282204/Put/seqid=0 2024-11-19T00:14:50,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742156_1332 (size=12301) 2024-11-19T00:14:50,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T00:14:50,841 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/8437a2db52374c96bd1b6ce7b003156e 2024-11-19T00:14:50,850 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/ea529c7242354a11957a1ffe4de4ee74 is 50, key is test_row_0/B:col10/1731975282204/Put/seqid=0 2024-11-19T00:14:50,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742157_1333 (size=12301) 2024-11-19T00:14:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T00:14:51,256 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/ea529c7242354a11957a1ffe4de4ee74 2024-11-19T00:14:51,262 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/a6ee03aa0d7c46a68715765b320eecb1 is 50, key is test_row_0/C:col10/1731975282204/Put/seqid=0 2024-11-19T00:14:51,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742158_1334 (size=12301) 2024-11-19T00:14:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T00:14:51,666 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/a6ee03aa0d7c46a68715765b320eecb1 2024-11-19T00:14:51,672 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/A/8437a2db52374c96bd1b6ce7b003156e as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/8437a2db52374c96bd1b6ce7b003156e 2024-11-19T00:14:51,677 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/8437a2db52374c96bd1b6ce7b003156e, entries=150, sequenceid=433, filesize=12.0 K 2024-11-19T00:14:51,678 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/B/ea529c7242354a11957a1ffe4de4ee74 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/ea529c7242354a11957a1ffe4de4ee74 2024-11-19T00:14:51,683 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/ea529c7242354a11957a1ffe4de4ee74, entries=150, sequenceid=433, filesize=12.0 K 2024-11-19T00:14:51,684 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/.tmp/C/a6ee03aa0d7c46a68715765b320eecb1 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/a6ee03aa0d7c46a68715765b320eecb1 2024-11-19T00:14:51,688 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/a6ee03aa0d7c46a68715765b320eecb1, entries=150, sequenceid=433, filesize=12.0 K 2024-11-19T00:14:51,689 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c24634f4d3f0bb64baf797330c06cc91 in 1256ms, sequenceid=433, compaction requested=true 2024-11-19T00:14:51,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/ecce0326ec564a7e95be9eb28e5e50a5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/23fcc7b6131a4af69dc5284cb5655190, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/1778d28317654c8ebb091cc8f1c14cbb, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/f790e812762345259e2ee48a9cd53740, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/d7264b9a285045418442df726ab991af, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/b6b9beafd5164293ba434e29c9a6a73a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/7b88b524b86d4ff7959f6bcd595b14f5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/0fb766752b8a48b590ba3908d9d9595c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/3d4379e470774227aca756a75b2136a3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/8a09ef74a64a42da8091933a61cc850b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/9adf9c898ac74cd99e49bebecc20f31b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/42bd0e6b1dc543518926eac6e0b57174, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/66b45029a71d483b8de47a6788f1db37, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/c9a86e7635c2471c9430097427f3c4dc, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/602b410387f3464a8a166fec614ae9fc, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/dd437c6a14284541b5e9d5f94326f4bd, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/f3b53aa8df4c4f03a667d2ad77e68d31, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/c82c498f6dca49b392a21085aa886b00, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/b654684f4b0d4df288130463d5124b91, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/42372bb7caeb450ebaa0387c50f540d5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/b7a7b6e1be8948b5a75b7ba27e6def55, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/95ace14d39e64dcea59e4ccf8ba43f08, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/c523adc21f1d41d88711869c6dc15497, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/098fa6991be54505b4e63959ce359ba9, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/bbce5b5b771b4a378f0960d07b6a1d5f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/863f1ee0c11d481e9e904b657ef6ced9, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/15e4f38a6d3b4c0fbcfe5b923763e24e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/e0d09775920146f0a640cb4e096b5088] to archive 2024-11-19T00:14:51,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T00:14:51,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/ecce0326ec564a7e95be9eb28e5e50a5 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/ecce0326ec564a7e95be9eb28e5e50a5 2024-11-19T00:14:51,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/23fcc7b6131a4af69dc5284cb5655190 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/23fcc7b6131a4af69dc5284cb5655190 2024-11-19T00:14:51,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/1778d28317654c8ebb091cc8f1c14cbb to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/1778d28317654c8ebb091cc8f1c14cbb 2024-11-19T00:14:51,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/f790e812762345259e2ee48a9cd53740 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/f790e812762345259e2ee48a9cd53740 2024-11-19T00:14:51,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/d7264b9a285045418442df726ab991af to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/d7264b9a285045418442df726ab991af 2024-11-19T00:14:51,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/b6b9beafd5164293ba434e29c9a6a73a to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/b6b9beafd5164293ba434e29c9a6a73a 2024-11-19T00:14:51,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/7b88b524b86d4ff7959f6bcd595b14f5 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/7b88b524b86d4ff7959f6bcd595b14f5 2024-11-19T00:14:51,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/0fb766752b8a48b590ba3908d9d9595c to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/0fb766752b8a48b590ba3908d9d9595c 2024-11-19T00:14:51,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/3d4379e470774227aca756a75b2136a3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/3d4379e470774227aca756a75b2136a3 2024-11-19T00:14:51,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/8a09ef74a64a42da8091933a61cc850b to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/8a09ef74a64a42da8091933a61cc850b 2024-11-19T00:14:51,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/9adf9c898ac74cd99e49bebecc20f31b to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/9adf9c898ac74cd99e49bebecc20f31b 2024-11-19T00:14:51,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/42bd0e6b1dc543518926eac6e0b57174 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/42bd0e6b1dc543518926eac6e0b57174 2024-11-19T00:14:51,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/66b45029a71d483b8de47a6788f1db37 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/66b45029a71d483b8de47a6788f1db37 2024-11-19T00:14:51,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/c9a86e7635c2471c9430097427f3c4dc to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/c9a86e7635c2471c9430097427f3c4dc 2024-11-19T00:14:51,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/602b410387f3464a8a166fec614ae9fc to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/602b410387f3464a8a166fec614ae9fc 2024-11-19T00:14:51,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/dd437c6a14284541b5e9d5f94326f4bd to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/dd437c6a14284541b5e9d5f94326f4bd 2024-11-19T00:14:51,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/f3b53aa8df4c4f03a667d2ad77e68d31 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/f3b53aa8df4c4f03a667d2ad77e68d31 2024-11-19T00:14:51,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/c82c498f6dca49b392a21085aa886b00 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/c82c498f6dca49b392a21085aa886b00 2024-11-19T00:14:51,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/b654684f4b0d4df288130463d5124b91 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/b654684f4b0d4df288130463d5124b91 2024-11-19T00:14:51,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/42372bb7caeb450ebaa0387c50f540d5 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/42372bb7caeb450ebaa0387c50f540d5 2024-11-19T00:14:51,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/b7a7b6e1be8948b5a75b7ba27e6def55 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/b7a7b6e1be8948b5a75b7ba27e6def55 2024-11-19T00:14:51,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/95ace14d39e64dcea59e4ccf8ba43f08 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/95ace14d39e64dcea59e4ccf8ba43f08 2024-11-19T00:14:51,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/c523adc21f1d41d88711869c6dc15497 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/c523adc21f1d41d88711869c6dc15497 2024-11-19T00:14:51,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/098fa6991be54505b4e63959ce359ba9 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/098fa6991be54505b4e63959ce359ba9 2024-11-19T00:14:51,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/bbce5b5b771b4a378f0960d07b6a1d5f to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/bbce5b5b771b4a378f0960d07b6a1d5f 2024-11-19T00:14:51,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/863f1ee0c11d481e9e904b657ef6ced9 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/863f1ee0c11d481e9e904b657ef6ced9 2024-11-19T00:14:51,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/15e4f38a6d3b4c0fbcfe5b923763e24e to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/15e4f38a6d3b4c0fbcfe5b923763e24e 2024-11-19T00:14:51,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/e0d09775920146f0a640cb4e096b5088 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/e0d09775920146f0a640cb4e096b5088 2024-11-19T00:14:51,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/1186e2e1d1f446a0874387e6eb317576, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/264f3b1854404e1a8de0ab7ff1117d51, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/e9b8beb74bb24d3781354eae3949123e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/99727e89f29c48fabc42ce946d7c1348, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/ef7293c84cac40ba809b656e50b425c5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/13f79f830f544e50b7dccc016b2352eb, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/333bf225b39041368b2fa4bc1d3eb78f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/59d3b08475fd4877b38b93867e7c8164, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/2e0e01dea1d741ec8e95966c872f9b83, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/cedead13538545f9b1778ea190ac714b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/9188c81d457947f088fbdf48415eb2a4, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/bc35b1d3aac14aa8bb57c94f11a90739, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/b1f8666a3b7b4cdda30046a836bb1562, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/81d5fdca304a41a2869f6a23bdff69f8, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/d3c41270738e43ef9c69652b620e8b94, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/f7dbdf918fa54d76b6ce8bfc0b753614, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/5da538c1c4cd4b68b8b2934b57868f5e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/9de960dc8a6d48809bb7361cf7a2fd89, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/91497e9fb362484da9bf7f14580d2189, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/4de46692345a46638ccd7f9c4de485a1, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/2e225a50553a447fa64e34c7b9556a5d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/7fb91553901f423887d5f96c36b05fad, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/6e4f09a89fc44dce9163d5e23481e552, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/4a5b2f6969cf4f65a2974a6e12fe0a1d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/1802c3a22d5147df9211791c850e5431, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/9af60292266149e287e88b036438bc74, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/d777b4ad7276461eb51eea9f1a779685, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/9f8b6dd6134d4d928d2425815203530d] to archive 2024-11-19T00:14:51,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T00:14:51,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/1186e2e1d1f446a0874387e6eb317576 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/1186e2e1d1f446a0874387e6eb317576 2024-11-19T00:14:51,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/264f3b1854404e1a8de0ab7ff1117d51 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/264f3b1854404e1a8de0ab7ff1117d51 2024-11-19T00:14:51,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/e9b8beb74bb24d3781354eae3949123e to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/e9b8beb74bb24d3781354eae3949123e 2024-11-19T00:14:51,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/99727e89f29c48fabc42ce946d7c1348 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/99727e89f29c48fabc42ce946d7c1348 2024-11-19T00:14:51,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/ef7293c84cac40ba809b656e50b425c5 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/ef7293c84cac40ba809b656e50b425c5 2024-11-19T00:14:51,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/13f79f830f544e50b7dccc016b2352eb to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/13f79f830f544e50b7dccc016b2352eb 2024-11-19T00:14:51,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/333bf225b39041368b2fa4bc1d3eb78f to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/333bf225b39041368b2fa4bc1d3eb78f 2024-11-19T00:14:51,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/59d3b08475fd4877b38b93867e7c8164 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/59d3b08475fd4877b38b93867e7c8164 2024-11-19T00:14:51,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/2e0e01dea1d741ec8e95966c872f9b83 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/2e0e01dea1d741ec8e95966c872f9b83 2024-11-19T00:14:51,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/cedead13538545f9b1778ea190ac714b to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/cedead13538545f9b1778ea190ac714b 2024-11-19T00:14:51,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/9188c81d457947f088fbdf48415eb2a4 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/9188c81d457947f088fbdf48415eb2a4 2024-11-19T00:14:51,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/bc35b1d3aac14aa8bb57c94f11a90739 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/bc35b1d3aac14aa8bb57c94f11a90739 2024-11-19T00:14:51,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/b1f8666a3b7b4cdda30046a836bb1562 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/b1f8666a3b7b4cdda30046a836bb1562 2024-11-19T00:14:51,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/81d5fdca304a41a2869f6a23bdff69f8 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/81d5fdca304a41a2869f6a23bdff69f8 2024-11-19T00:14:51,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/d3c41270738e43ef9c69652b620e8b94 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/d3c41270738e43ef9c69652b620e8b94 2024-11-19T00:14:51,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/f7dbdf918fa54d76b6ce8bfc0b753614 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/f7dbdf918fa54d76b6ce8bfc0b753614 2024-11-19T00:14:51,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/5da538c1c4cd4b68b8b2934b57868f5e to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/5da538c1c4cd4b68b8b2934b57868f5e 2024-11-19T00:14:51,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/9de960dc8a6d48809bb7361cf7a2fd89 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/9de960dc8a6d48809bb7361cf7a2fd89 2024-11-19T00:14:51,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/91497e9fb362484da9bf7f14580d2189 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/91497e9fb362484da9bf7f14580d2189 2024-11-19T00:14:51,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/4de46692345a46638ccd7f9c4de485a1 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/4de46692345a46638ccd7f9c4de485a1 2024-11-19T00:14:51,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/2e225a50553a447fa64e34c7b9556a5d to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/2e225a50553a447fa64e34c7b9556a5d 2024-11-19T00:14:51,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/7fb91553901f423887d5f96c36b05fad to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/7fb91553901f423887d5f96c36b05fad 2024-11-19T00:14:51,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/6e4f09a89fc44dce9163d5e23481e552 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/6e4f09a89fc44dce9163d5e23481e552 2024-11-19T00:14:51,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/4a5b2f6969cf4f65a2974a6e12fe0a1d to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/4a5b2f6969cf4f65a2974a6e12fe0a1d 2024-11-19T00:14:51,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/1802c3a22d5147df9211791c850e5431 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/1802c3a22d5147df9211791c850e5431 2024-11-19T00:14:51,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/9af60292266149e287e88b036438bc74 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/9af60292266149e287e88b036438bc74 2024-11-19T00:14:51,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/d777b4ad7276461eb51eea9f1a779685 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/d777b4ad7276461eb51eea9f1a779685 2024-11-19T00:14:51,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/9f8b6dd6134d4d928d2425815203530d to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/9f8b6dd6134d4d928d2425815203530d 2024-11-19T00:14:51,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/37dcd6b9f793442184c6e5a4791f4477, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/3eb2cc4269514444b3520a23ea2974c3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/439b1b13dca248df95b5c84ec7237ffe, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/d6034cf499384a2aaf2d986912db4bb0, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/dbaa71148d094e699b565b4cc8b0df9f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/52a356004a474a42a72ef96327a0e62c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/ac12de8fac1041b1b1cf1fd9732d8077, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/35d61a2ff6954c83be82ce6ae9e64053, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/b978b23e9915475ca4eb91419c2bb8d1, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/9a919190a442441a8f6d2596dafe2e59, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/80a2f833fa774711bbf06fae57e2905a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/44a04929820f4aacbcf9555fd209fefc, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/a03d87df29394e2bb0173629de4fec3b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/5b84d6fc9573419aad9d7c5b1c8a8658, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/09e5e74a037848d1993b01eeef08d996, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/6cfc079d09304998b3aaae783145bae5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/f8850ddfa0884be8b78c0ec74526c279, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/f411809aafb04580b87edb43f7d8d253, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/80e9f7522c854460a239be04ac05572b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/65b8d4472334473b9c10e061d31527f9, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/decb60b2d4024e0eaace1ca5324f0bdb, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/546c8d7569f54cdf92a405f5890f9564, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/da9e5058369f4a63a79d967f0cb6371e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/4eb06bd2f2194bd08d973ead1b74568a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/cd2628dcecd349559580b9f7ae38dbca, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/d5558fb544ce4f2789cd302dcf71019e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/eb87fc7b572043e8bb561078372b61ca, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/4b22e7a123134d758121709e8b1a2d31] to archive 2024-11-19T00:14:51,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T00:14:51,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/37dcd6b9f793442184c6e5a4791f4477 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/37dcd6b9f793442184c6e5a4791f4477 2024-11-19T00:14:51,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/3eb2cc4269514444b3520a23ea2974c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/3eb2cc4269514444b3520a23ea2974c3 2024-11-19T00:14:51,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/439b1b13dca248df95b5c84ec7237ffe to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/439b1b13dca248df95b5c84ec7237ffe 2024-11-19T00:14:51,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/d6034cf499384a2aaf2d986912db4bb0 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/d6034cf499384a2aaf2d986912db4bb0 2024-11-19T00:14:51,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/dbaa71148d094e699b565b4cc8b0df9f to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/dbaa71148d094e699b565b4cc8b0df9f 2024-11-19T00:14:51,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/52a356004a474a42a72ef96327a0e62c to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/52a356004a474a42a72ef96327a0e62c 2024-11-19T00:14:51,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/ac12de8fac1041b1b1cf1fd9732d8077 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/ac12de8fac1041b1b1cf1fd9732d8077 2024-11-19T00:14:51,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/35d61a2ff6954c83be82ce6ae9e64053 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/35d61a2ff6954c83be82ce6ae9e64053 2024-11-19T00:14:51,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/b978b23e9915475ca4eb91419c2bb8d1 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/b978b23e9915475ca4eb91419c2bb8d1 2024-11-19T00:14:51,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/9a919190a442441a8f6d2596dafe2e59 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/9a919190a442441a8f6d2596dafe2e59 2024-11-19T00:14:51,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/80a2f833fa774711bbf06fae57e2905a to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/80a2f833fa774711bbf06fae57e2905a 2024-11-19T00:14:51,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/44a04929820f4aacbcf9555fd209fefc to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/44a04929820f4aacbcf9555fd209fefc 2024-11-19T00:14:51,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/a03d87df29394e2bb0173629de4fec3b to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/a03d87df29394e2bb0173629de4fec3b 2024-11-19T00:14:51,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/5b84d6fc9573419aad9d7c5b1c8a8658 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/5b84d6fc9573419aad9d7c5b1c8a8658 2024-11-19T00:14:51,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/09e5e74a037848d1993b01eeef08d996 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/09e5e74a037848d1993b01eeef08d996 2024-11-19T00:14:51,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/6cfc079d09304998b3aaae783145bae5 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/6cfc079d09304998b3aaae783145bae5 2024-11-19T00:14:51,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/f8850ddfa0884be8b78c0ec74526c279 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/f8850ddfa0884be8b78c0ec74526c279 2024-11-19T00:14:51,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/f411809aafb04580b87edb43f7d8d253 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/f411809aafb04580b87edb43f7d8d253 2024-11-19T00:14:51,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/80e9f7522c854460a239be04ac05572b to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/80e9f7522c854460a239be04ac05572b 2024-11-19T00:14:51,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/65b8d4472334473b9c10e061d31527f9 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/65b8d4472334473b9c10e061d31527f9 2024-11-19T00:14:51,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/decb60b2d4024e0eaace1ca5324f0bdb to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/decb60b2d4024e0eaace1ca5324f0bdb 2024-11-19T00:14:51,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/546c8d7569f54cdf92a405f5890f9564 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/546c8d7569f54cdf92a405f5890f9564 2024-11-19T00:14:51,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/da9e5058369f4a63a79d967f0cb6371e to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/da9e5058369f4a63a79d967f0cb6371e 2024-11-19T00:14:51,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/4eb06bd2f2194bd08d973ead1b74568a to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/4eb06bd2f2194bd08d973ead1b74568a 2024-11-19T00:14:51,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/cd2628dcecd349559580b9f7ae38dbca to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/cd2628dcecd349559580b9f7ae38dbca 2024-11-19T00:14:51,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/d5558fb544ce4f2789cd302dcf71019e to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/d5558fb544ce4f2789cd302dcf71019e 2024-11-19T00:14:51,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/eb87fc7b572043e8bb561078372b61ca to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/eb87fc7b572043e8bb561078372b61ca 2024-11-19T00:14:51,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/4b22e7a123134d758121709e8b1a2d31 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/4b22e7a123134d758121709e8b1a2d31 2024-11-19T00:14:51,787 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/recovered.edits/436.seqid, newMaxSeqId=436, maxSeqId=1 2024-11-19T00:14:51,788 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. 2024-11-19T00:14:51,788 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for c24634f4d3f0bb64baf797330c06cc91: Waiting for close lock at 1731975290433Running coprocessor pre-close hooks at 1731975290433Disabling compacts and flushes for region at 1731975290433Disabling writes for close at 1731975290433Obtaining lock to block concurrent updates at 1731975290433Preparing flush snapshotting stores in c24634f4d3f0bb64baf797330c06cc91 at 1731975290433Finished memstore snapshotting TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731975290434 (+1 ms)Flushing stores of TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91. at 1731975290434Flushing c24634f4d3f0bb64baf797330c06cc91/A: creating writer at 1731975290434Flushing c24634f4d3f0bb64baf797330c06cc91/A: appending metadata at 1731975290436 (+2 ms)Flushing c24634f4d3f0bb64baf797330c06cc91/A: closing flushed file at 1731975290436Flushing c24634f4d3f0bb64baf797330c06cc91/B: creating writer at 1731975290846 (+410 ms)Flushing c24634f4d3f0bb64baf797330c06cc91/B: appending metadata at 1731975290849 (+3 ms)Flushing c24634f4d3f0bb64baf797330c06cc91/B: closing flushed file at 1731975290849Flushing c24634f4d3f0bb64baf797330c06cc91/C: creating writer at 1731975291259 (+410 ms)Flushing c24634f4d3f0bb64baf797330c06cc91/C: appending metadata at 1731975291262 (+3 ms)Flushing c24634f4d3f0bb64baf797330c06cc91/C: closing flushed file at 1731975291262Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d1aaeda: reopening flushed file at 1731975291671 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f87b37b: reopening flushed file at 1731975291677 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37d8ba58: reopening flushed file at 1731975291683 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c24634f4d3f0bb64baf797330c06cc91 in 1256ms, sequenceid=433, compaction requested=true at 1731975291689 (+6 ms)Writing region close event to WAL at 1731975291784 (+95 ms)Running coprocessor post-close hooks at 1731975291787 (+3 ms)Closed at 1731975291787 2024-11-19T00:14:51,790 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:51,790 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=c24634f4d3f0bb64baf797330c06cc91, regionState=CLOSED 2024-11-19T00:14:51,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 because future has completed 2024-11-19T00:14:51,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-19T00:14:51,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure c24634f4d3f0bb64baf797330c06cc91, server=6b1531871d35,38567,1731975194317 in 1.5150 sec 2024-11-19T00:14:51,798 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-19T00:14:51,798 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c24634f4d3f0bb64baf797330c06cc91, UNASSIGN in 1.5200 sec 2024-11-19T00:14:51,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-19T00:14:51,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5250 sec 2024-11-19T00:14:51,802 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975291801"}]},"ts":"1731975291801"} 2024-11-19T00:14:51,803 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T00:14:51,803 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T00:14:51,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5360 sec 2024-11-19T00:14:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T00:14:52,424 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:14:52,425 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T00:14:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:14:52,427 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:14:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T00:14:52,428 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:14:52,430 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:52,432 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A, FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B, FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C, FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/recovered.edits] 2024-11-19T00:14:52,434 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/018cf872516e48d2ac57a743907fdab6 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/018cf872516e48d2ac57a743907fdab6 2024-11-19T00:14:52,435 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/33b43e1ea7fd493ba5f4ddc1d4d992fc to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/33b43e1ea7fd493ba5f4ddc1d4d992fc 2024-11-19T00:14:52,436 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/8437a2db52374c96bd1b6ce7b003156e to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/8437a2db52374c96bd1b6ce7b003156e 2024-11-19T00:14:52,438 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/b6f84613e39e49cbbedb93fcff03a95e to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/A/b6f84613e39e49cbbedb93fcff03a95e 2024-11-19T00:14:52,440 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/3da476f3b70848539e0996c6aeaf01ef to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/3da476f3b70848539e0996c6aeaf01ef 2024-11-19T00:14:52,441 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/b982449a36444ea4a2a7901869b1195b to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/b982449a36444ea4a2a7901869b1195b 2024-11-19T00:14:52,442 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/ea529c7242354a11957a1ffe4de4ee74 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/ea529c7242354a11957a1ffe4de4ee74 2024-11-19T00:14:52,443 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/f7e00fc92ee7497eb3765bff1ff009a8 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/B/f7e00fc92ee7497eb3765bff1ff009a8 2024-11-19T00:14:52,445 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/0a0db085677245d78f9f67d93022ea80 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/0a0db085677245d78f9f67d93022ea80 2024-11-19T00:14:52,445 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/a6ee03aa0d7c46a68715765b320eecb1 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/a6ee03aa0d7c46a68715765b320eecb1 2024-11-19T00:14:52,447 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/d0bd11ccff404533a7ca963a02a72b91 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/d0bd11ccff404533a7ca963a02a72b91 2024-11-19T00:14:52,447 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/f944ba86ea22493890b7652873f014de to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/C/f944ba86ea22493890b7652873f014de 2024-11-19T00:14:52,450 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/recovered.edits/436.seqid to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91/recovered.edits/436.seqid 2024-11-19T00:14:52,450 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/c24634f4d3f0bb64baf797330c06cc91 2024-11-19T00:14:52,450 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T00:14:52,452 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:14:52,456 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T00:14:52,458 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T00:14:52,459 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:14:52,459 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T00:14:52,459 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731975292459"}]},"ts":"9223372036854775807"} 2024-11-19T00:14:52,461 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T00:14:52,461 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c24634f4d3f0bb64baf797330c06cc91, NAME => 'TestAcidGuarantees,,1731975259606.c24634f4d3f0bb64baf797330c06cc91.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T00:14:52,461 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T00:14:52,462 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731975292461"}]},"ts":"9223372036854775807"} 2024-11-19T00:14:52,464 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T00:14:52,465 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:14:52,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 40 msec 2024-11-19T00:14:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T00:14:52,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T00:14:52,535 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:14:52,544 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 248), OpenFileDescriptor=447 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=495 (was 460) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1931 (was 2461) 2024-11-19T00:14:52,553 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=495, ProcessCount=11, AvailableMemoryMB=1932 2024-11-19T00:14:52,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T00:14:52,555 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T00:14:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T00:14:52,557 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T00:14:52,557 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:52,558 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-11-19T00:14:52,558 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T00:14:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-19T00:14:52,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742159_1335 (size=1061) 2024-11-19T00:14:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-19T00:14:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-19T00:14:52,965 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2 2024-11-19T00:14:52,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742160_1336 (size=53) 2024-11-19T00:14:53,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-19T00:14:53,370 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:14:53,371 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 999ebf86880fba150650b0d98c4af3c3, disabling compactions & flushes 2024-11-19T00:14:53,371 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:53,371 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:53,371 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. after waiting 0 ms 2024-11-19T00:14:53,371 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:53,371 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:53,371 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 999ebf86880fba150650b0d98c4af3c3: Waiting for close lock at 1731975293371Disabling compacts and flushes for region at 1731975293371Disabling writes for close at 1731975293371Writing region close event to WAL at 1731975293371Closed at 1731975293371 2024-11-19T00:14:53,372 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T00:14:53,372 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731975293372"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731975293372"}]},"ts":"1731975293372"} 2024-11-19T00:14:53,374 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T00:14:53,375 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T00:14:53,375 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975293375"}]},"ts":"1731975293375"} 2024-11-19T00:14:53,377 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T00:14:53,377 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=999ebf86880fba150650b0d98c4af3c3, ASSIGN}] 2024-11-19T00:14:53,378 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=999ebf86880fba150650b0d98c4af3c3, ASSIGN 2024-11-19T00:14:53,379 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=999ebf86880fba150650b0d98c4af3c3, ASSIGN; state=OFFLINE, location=6b1531871d35,38567,1731975194317; forceNewPlan=false, retain=false 2024-11-19T00:14:53,530 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=999ebf86880fba150650b0d98c4af3c3, regionState=OPENING, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:14:53,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=999ebf86880fba150650b0d98c4af3c3, ASSIGN because future has completed 2024-11-19T00:14:53,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317}] 2024-11-19T00:14:53,689 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:53,689 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} 2024-11-19T00:14:53,690 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:53,690 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:14:53,690 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:53,690 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:53,692 INFO [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:53,693 INFO [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:14:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-19T00:14:53,694 INFO [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 999ebf86880fba150650b0d98c4af3c3 columnFamilyName A 2024-11-19T00:14:53,694 DEBUG [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:53,695 INFO [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] regionserver.HStore(327): Store=999ebf86880fba150650b0d98c4af3c3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:14:53,695 INFO [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:53,696 INFO [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:14:53,696 INFO [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 999ebf86880fba150650b0d98c4af3c3 columnFamilyName B 2024-11-19T00:14:53,696 DEBUG [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:53,697 INFO [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] regionserver.HStore(327): Store=999ebf86880fba150650b0d98c4af3c3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:14:53,697 INFO [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:53,698 INFO [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:14:53,698 INFO [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 999ebf86880fba150650b0d98c4af3c3 columnFamilyName C 2024-11-19T00:14:53,698 DEBUG [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:53,699 INFO [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] regionserver.HStore(327): Store=999ebf86880fba150650b0d98c4af3c3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:14:53,699 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:53,699 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:53,700 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:53,700 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:53,702 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:53,702 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:53,702 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T00:14:53,704 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:53,706 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T00:14:53,706 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened 999ebf86880fba150650b0d98c4af3c3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60905784, jitterRate=-0.09243309497833252}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T00:14:53,706 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:53,707 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for 999ebf86880fba150650b0d98c4af3c3: Running coprocessor pre-open hook at 1731975293690Writing region info on filesystem at 1731975293690Initializing all the Stores at 1731975293691 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975293691Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975293692 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975293692Cleaning up temporary data from old regions at 1731975293702 (+10 ms)Running coprocessor post-open hooks at 1731975293706 (+4 ms)Region opened successfully at 1731975293707 (+1 ms) 2024-11-19T00:14:53,708 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., pid=87, masterSystemTime=1731975293685 2024-11-19T00:14:53,710 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:53,710 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:53,711 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=999ebf86880fba150650b0d98c4af3c3, regionState=OPEN, openSeqNum=2, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:14:53,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 because future has completed 2024-11-19T00:14:53,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-19T00:14:53,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 in 181 msec 2024-11-19T00:14:53,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-19T00:14:53,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=999ebf86880fba150650b0d98c4af3c3, ASSIGN in 338 msec 2024-11-19T00:14:53,717 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T00:14:53,718 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975293717"}]},"ts":"1731975293717"} 2024-11-19T00:14:53,719 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T00:14:53,720 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T00:14:53,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1650 sec 2024-11-19T00:14:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-19T00:14:54,704 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:14:54,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197fefe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:54,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:14:54,706 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:14:54,707 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:14:54,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:14:54,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:14:54,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d07e2bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:54,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:14:54,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:14:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:54,709 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:14:54,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fc5faa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:54,710 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:14:54,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:14:54,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:14:54,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:14:54,713 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T00:14:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 6b1531871d35,33843,1731975193570 2024-11-19T00:14:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@cd8adf3 2024-11-19T00:14:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T00:14:54,715 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T00:14:54,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T00:14:54,716 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T00:14:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T00:14:54,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742161_1337 (size=1097) 2024-11-19T00:14:55,130 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-19T00:14:55,130 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-19T00:14:55,132 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T00:14:55,134 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=999ebf86880fba150650b0d98c4af3c3, REOPEN/MOVE}] 2024-11-19T00:14:55,135 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=999ebf86880fba150650b0d98c4af3c3, REOPEN/MOVE 2024-11-19T00:14:55,136 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=999ebf86880fba150650b0d98c4af3c3, regionState=CLOSING, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:14:55,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=999ebf86880fba150650b0d98c4af3c3, REOPEN/MOVE because future has completed 2024-11-19T00:14:55,137 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T00:14:55,137 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317}] 2024-11-19T00:14:55,290 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:55,290 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T00:14:55,290 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 999ebf86880fba150650b0d98c4af3c3, disabling compactions & flushes 2024-11-19T00:14:55,290 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:55,290 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:55,290 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. after waiting 0 ms 2024-11-19T00:14:55,290 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:55,294 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T00:14:55,295 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:55,295 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 999ebf86880fba150650b0d98c4af3c3: Waiting for close lock at 1731975295290Running coprocessor pre-close hooks at 1731975295290Disabling compacts and flushes for region at 1731975295290Disabling writes for close at 1731975295290Writing region close event to WAL at 1731975295291 (+1 ms)Running coprocessor post-close hooks at 1731975295295 (+4 ms)Closed at 1731975295295 2024-11-19T00:14:55,295 WARN [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: 999ebf86880fba150650b0d98c4af3c3 to self. 2024-11-19T00:14:55,297 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:55,298 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=999ebf86880fba150650b0d98c4af3c3, regionState=CLOSED 2024-11-19T00:14:55,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 because future has completed 2024-11-19T00:14:55,301 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-19T00:14:55,301 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 in 163 msec 2024-11-19T00:14:55,302 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=999ebf86880fba150650b0d98c4af3c3, REOPEN/MOVE; state=CLOSED, location=6b1531871d35,38567,1731975194317; forceNewPlan=false, retain=true 2024-11-19T00:14:55,452 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=999ebf86880fba150650b0d98c4af3c3, regionState=OPENING, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:14:55,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=999ebf86880fba150650b0d98c4af3c3, REOPEN/MOVE because future has completed 2024-11-19T00:14:55,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317}] 2024-11-19T00:14:55,609 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:55,609 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} 2024-11-19T00:14:55,609 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:55,609 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:14:55,610 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:55,610 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:55,611 INFO [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:55,612 INFO [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:14:55,612 INFO [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 999ebf86880fba150650b0d98c4af3c3 columnFamilyName A 2024-11-19T00:14:55,613 DEBUG [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:55,613 INFO [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] regionserver.HStore(327): Store=999ebf86880fba150650b0d98c4af3c3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:14:55,614 INFO [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:55,614 INFO [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:14:55,614 INFO [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 999ebf86880fba150650b0d98c4af3c3 columnFamilyName B 2024-11-19T00:14:55,614 DEBUG [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:55,615 INFO [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] regionserver.HStore(327): Store=999ebf86880fba150650b0d98c4af3c3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:14:55,615 INFO [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:55,615 INFO [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:14:55,615 INFO [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 999ebf86880fba150650b0d98c4af3c3 columnFamilyName C 2024-11-19T00:14:55,615 DEBUG [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:55,616 INFO [StoreOpener-999ebf86880fba150650b0d98c4af3c3-1 {}] regionserver.HStore(327): Store=999ebf86880fba150650b0d98c4af3c3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:14:55,616 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:55,616 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:55,616 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:55,617 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:55,618 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:55,618 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:55,619 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T00:14:55,620 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:55,621 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened 999ebf86880fba150650b0d98c4af3c3; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66157707, jitterRate=-0.014173343777656555}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T00:14:55,621 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:55,621 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for 999ebf86880fba150650b0d98c4af3c3: Running coprocessor pre-open hook at 1731975295610Writing region info on filesystem at 1731975295610Initializing all the Stores at 1731975295611 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975295611Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975295611Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975295611Cleaning up temporary data from old regions at 1731975295618 (+7 ms)Running coprocessor post-open hooks at 1731975295621 (+3 ms)Region opened successfully at 1731975295621 2024-11-19T00:14:55,622 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., pid=92, masterSystemTime=1731975295606 2024-11-19T00:14:55,624 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:55,624 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:55,624 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=999ebf86880fba150650b0d98c4af3c3, regionState=OPEN, openSeqNum=5, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:14:55,625 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 because future has completed 2024-11-19T00:14:55,627 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-11-19T00:14:55,627 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 in 172 msec 2024-11-19T00:14:55,629 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-19T00:14:55,629 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=999ebf86880fba150650b0d98c4af3c3, REOPEN/MOVE in 493 msec 2024-11-19T00:14:55,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-19T00:14:55,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 497 msec 2024-11-19T00:14:55,634 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 915 msec 2024-11-19T00:14:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T00:14:55,635 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:14:55,635 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,635 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:14:55,636 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:14:55,636 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:14:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:14:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:14:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27819e6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:14:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:14:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:55,638 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:14:55,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a864b23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,639 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:14:55,639 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:14:55,640 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:14:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:14:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:14:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ef0d7ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:14:55,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:14:55,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:55,641 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:14:55,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@618fe230, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,643 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:14:55,643 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:14:55,644 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:14:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:14:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:14:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cf136a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:14:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:14:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:55,645 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:14:55,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6479474f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,646 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:14:55,646 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:14:55,647 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:14:55,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:14:55,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:14:55,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c20d116, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:14:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:14:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:55,649 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:14:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19f9df80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c86b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,650 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:14:55,650 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:14:55,651 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:14:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:14:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:14:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26cde7f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:14:55,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:14:55,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:55,652 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:14:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3badf4be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,654 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d098cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,654 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:14:55,654 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:14:55,655 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:14:55,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:14:55,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:14:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c6b386b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:14:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:14:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:55,657 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:14:55,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ab178c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,658 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:14:55,658 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:14:55,659 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:14:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:14:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:14:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@378f2670, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:14:55,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:14:55,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:55,660 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:14:55,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55cd97ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,662 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:14:55,662 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:14:55,663 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:14:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:14:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:14:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@756fbf94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:14:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:14:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:55,664 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:14:55,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ea34d62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,665 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:14:55,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:14:55,666 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:14:55,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:14:55,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:14:55,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@316eeadb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:14:55,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:14:55,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:55,668 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:14:55,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41898c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d10c554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:14:55,670 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:14:55,671 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:14:55,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:14:55,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:14:55,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37131852, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:14:55,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:14:55,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:14:55,672 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:14:55,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4acc70cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:14:55,674 DEBUG [Thread-1553 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:14:55,674 DEBUG [Thread-1555 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:14:55,674 DEBUG [Thread-1542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:14:55,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:14:55,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:14:55,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:14:55,675 DEBUG [Thread-1551 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:14:55,675 DEBUG [Thread-1548 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:14:55,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:14:55,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:14:55,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:14:55,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:14:55,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:14:55,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:14:55,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:14:55,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:14:55,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:14:55,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:14:55,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:14:55,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:14:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:14:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:14:55,681 DEBUG [Thread-1559 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:14:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:14:55,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:14:55,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:14:55,683 DEBUG [Thread-1559 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:14:55,683 DEBUG [Thread-1559 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:14:55,685 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:14:55,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:14:55,687 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:14:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-19T00:14:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T00:14:55,689 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:14:55,690 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:14:55,690 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:14:55,691 DEBUG [Thread-1546 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:14:55,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:14:55,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:14:55,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:14:55,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:14:55,696 DEBUG [Thread-1540 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:14:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:55,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 999ebf86880fba150650b0d98c4af3c3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T00:14:55,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=A 2024-11-19T00:14:55,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:55,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=B 2024-11-19T00:14:55,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:55,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=C 2024-11-19T00:14:55,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:55,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:14:55,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:14:55,700 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:14:55,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:14:55,711 DEBUG [Thread-1544 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:14:55,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:14:55,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:14:55,714 DEBUG [Thread-1557 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:14:55,715 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:14:55,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:14:55,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:14:55,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:14:55,718 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:14:55,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:14:55,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e245eece7aa84ee5991ee66ca1252ad1_999ebf86880fba150650b0d98c4af3c3 is 50, key is test_row_0/A:col10/1731975295685/Put/seqid=0 2024-11-19T00:14:55,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731975295710, max timestamp=1731975295737Num uniques 859; 2024-11-19T00:14:55,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731975295710, max timestamp=1731975295737Num uniques 859; 2024-11-19T00:14:55,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731975295710, max timestamp=1731975295737Num uniques 859; 2024-11-19T00:14:55,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T00:14:55,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T00:14:55,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T00:14:55,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742162_1338 (size=14594) 2024-11-19T00:14:55,755 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:55,761 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e245eece7aa84ee5991ee66ca1252ad1_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e245eece7aa84ee5991ee66ca1252ad1_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:55,762 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/ef00bb5e8f284f3a8c8e2db827b39993, store: [table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:14:55,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/ef00bb5e8f284f3a8c8e2db827b39993 is 175, key is test_row_0/A:col10/1731975295685/Put/seqid=0 2024-11-19T00:14:55,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742163_1339 (size=39549) 2024-11-19T00:14:55,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975355774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:55,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975355774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:55,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975355775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:55,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:55,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:55,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:55,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:55,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975355775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975355777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:55,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:55,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:55,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:55,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:55,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:55,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:55,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:55,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T00:14:55,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:55,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:55,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:55,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:55,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T00:14:55,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:55,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:14:55,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:55,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:55,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:55,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975355894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:55,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975355903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:55,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:55,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:55,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:55,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975355903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:55,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:55,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:55,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975355903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:55,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:55,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:55,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:55,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:55,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975355903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:55,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:55,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:55,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:55,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T00:14:55,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:55,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:14:55,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:55,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:55,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T00:14:56,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975356113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:56,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:56,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975356114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:56,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:56,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975356114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:56,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:56,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975356114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975356114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:56,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T00:14:56,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:56,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:14:56,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:56,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:56,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:56,166 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/ef00bb5e8f284f3a8c8e2db827b39993 2024-11-19T00:14:56,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/429f2da816ab4ccd94da6e0cec887dd6 is 50, key is test_row_0/B:col10/1731975295685/Put/seqid=0 2024-11-19T00:14:56,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742164_1340 (size=12001) 2024-11-19T00:14:56,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T00:14:56,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:56,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:14:56,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:56,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:56,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T00:14:56,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:56,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975356423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:56,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:56,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:56,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975356423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:56,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:56,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975356434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975356434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975356434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:56,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:56,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T00:14:56,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:56,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:14:56,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:56,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:56,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:56,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:56,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T00:14:56,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:56,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:14:56,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:56,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:56,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:56,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/429f2da816ab4ccd94da6e0cec887dd6 2024-11-19T00:14:56,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/ba260e839d2c4bbc8717ef9bd54d4fd6 is 50, key is test_row_0/C:col10/1731975295685/Put/seqid=0 2024-11-19T00:14:56,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742165_1341 (size=12001) 2024-11-19T00:14:56,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T00:14:56,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:56,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:14:56,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:56,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:56,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T00:14:56,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T00:14:56,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:56,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:14:56,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:56,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:56,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:14:56,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975356944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:56,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975356944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:56,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975356954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:56,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:56,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:56,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975356955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:56,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:56,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975356955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:56,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:56,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:57,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/ba260e839d2c4bbc8717ef9bd54d4fd6 2024-11-19T00:14:57,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/ef00bb5e8f284f3a8c8e2db827b39993 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/ef00bb5e8f284f3a8c8e2db827b39993 2024-11-19T00:14:57,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/ef00bb5e8f284f3a8c8e2db827b39993, entries=200, sequenceid=14, filesize=38.6 K 2024-11-19T00:14:57,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/429f2da816ab4ccd94da6e0cec887dd6 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/429f2da816ab4ccd94da6e0cec887dd6 2024-11-19T00:14:57,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/429f2da816ab4ccd94da6e0cec887dd6, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T00:14:57,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/ba260e839d2c4bbc8717ef9bd54d4fd6 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/ba260e839d2c4bbc8717ef9bd54d4fd6 2024-11-19T00:14:57,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/ba260e839d2c4bbc8717ef9bd54d4fd6, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T00:14:57,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 999ebf86880fba150650b0d98c4af3c3 in 1357ms, sequenceid=14, compaction requested=false 2024-11-19T00:14:57,054 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T00:14:57,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:14:57,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T00:14:57,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:57,066 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 999ebf86880fba150650b0d98c4af3c3 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-19T00:14:57,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=A 2024-11-19T00:14:57,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:14:57,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=B 2024-11-19T00:14:57,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:14:57,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=C 2024-11-19T00:14:57,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:14:57,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e96d8b98b9c8467fb9ff495e27182fb8_999ebf86880fba150650b0d98c4af3c3 is 50, key is test_row_0/A:col10/1731975295774/Put/seqid=0 2024-11-19T00:14:57,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742166_1342 (size=12154) 2024-11-19T00:14:57,316 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T00:14:57,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:57,490 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e96d8b98b9c8467fb9ff495e27182fb8_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e96d8b98b9c8467fb9ff495e27182fb8_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:57,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/1d562e9d38964d0fa7b4e547d7aaa824, store: [table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:14:57,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/1d562e9d38964d0fa7b4e547d7aaa824 is 175, key is test_row_0/A:col10/1731975295774/Put/seqid=0 2024-11-19T00:14:57,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742167_1343 (size=30955) 2024-11-19T00:14:57,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T00:14:57,937 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/1d562e9d38964d0fa7b4e547d7aaa824 2024-11-19T00:14:57,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/c9b19a9cf190495eb409f9312aab9358 is 50, key is test_row_0/B:col10/1731975295774/Put/seqid=0 2024-11-19T00:14:57,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:14:57,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:57,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742168_1344 (size=12001) 2024-11-19T00:14:57,979 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/c9b19a9cf190495eb409f9312aab9358 2024-11-19T00:14:57,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/fccf6507d2804855b4511c76ad009113 is 50, key is test_row_0/C:col10/1731975295774/Put/seqid=0 2024-11-19T00:14:58,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975357996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:58,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975358000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:58,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742169_1345 (size=12001) 2024-11-19T00:14:58,029 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/fccf6507d2804855b4511c76ad009113 2024-11-19T00:14:58,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975358004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:58,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975358006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:58,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975358001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:58,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/1d562e9d38964d0fa7b4e547d7aaa824 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/1d562e9d38964d0fa7b4e547d7aaa824 2024-11-19T00:14:58,044 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/1d562e9d38964d0fa7b4e547d7aaa824, entries=150, sequenceid=49, filesize=30.2 K 2024-11-19T00:14:58,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/c9b19a9cf190495eb409f9312aab9358 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/c9b19a9cf190495eb409f9312aab9358 2024-11-19T00:14:58,050 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/c9b19a9cf190495eb409f9312aab9358, entries=150, sequenceid=49, filesize=11.7 K 2024-11-19T00:14:58,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/fccf6507d2804855b4511c76ad009113 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/fccf6507d2804855b4511c76ad009113 2024-11-19T00:14:58,057 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/fccf6507d2804855b4511c76ad009113, entries=150, sequenceid=49, filesize=11.7 K 2024-11-19T00:14:58,058 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for 999ebf86880fba150650b0d98c4af3c3 in 992ms, sequenceid=49, compaction requested=false 2024-11-19T00:14:58,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:14:58,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:58,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-19T00:14:58,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-19T00:14:58,062 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-19T00:14:58,062 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3700 sec 2024-11-19T00:14:58,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.3760 sec 2024-11-19T00:14:58,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:58,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 999ebf86880fba150650b0d98c4af3c3 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T00:14:58,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=A 2024-11-19T00:14:58,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:58,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=B 2024-11-19T00:14:58,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:58,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=C 2024-11-19T00:14:58,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:14:58,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bdcd46a9db6e4ae8b77858015554092a_999ebf86880fba150650b0d98c4af3c3 is 50, key is test_row_0/A:col10/1731975298004/Put/seqid=0 2024-11-19T00:14:58,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742170_1346 (size=14594) 2024-11-19T00:14:58,150 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:58,157 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bdcd46a9db6e4ae8b77858015554092a_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bdcd46a9db6e4ae8b77858015554092a_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:58,158 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/1b401825aad64b7a8f078109a2d5d6b5, store: [table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:14:58,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/1b401825aad64b7a8f078109a2d5d6b5 is 175, key is test_row_0/A:col10/1731975298004/Put/seqid=0 2024-11-19T00:14:58,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742171_1347 (size=39549) 2024-11-19T00:14:58,172 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/1b401825aad64b7a8f078109a2d5d6b5 2024-11-19T00:14:58,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/1585c43935ae4b75ba3e05e01e614e94 is 50, key is test_row_0/B:col10/1731975298004/Put/seqid=0 2024-11-19T00:14:58,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731975298132, max timestamp=1731975298228Num uniques 859; 2024-11-19T00:14:58,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T00:14:58,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731975298132, max timestamp=1731975298228Num uniques 859; 2024-11-19T00:14:58,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T00:14:58,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742172_1348 (size=12001) 2024-11-19T00:14:58,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975298166, max timestamp=1731975298228Num uniques 859; 2024-11-19T00:14:58,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:14:58,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975358276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975358281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:58,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975358283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:58,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975358286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:58,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975358287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:58,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975358394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:58,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975358406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975358414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:58,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975358414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975358414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:58,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/1585c43935ae4b75ba3e05e01e614e94 2024-11-19T00:14:58,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975358618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/d451237c443c4238909bba1becdbb286 is 50, key is test_row_0/C:col10/1731975298004/Put/seqid=0 2024-11-19T00:14:58,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975358634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975358634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:58,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:58,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975358634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:58,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975358635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:58,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:58,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742173_1349 (size=12001) 2024-11-19T00:14:58,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975358954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:58,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975358954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:58,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975358954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:58,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975358954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:58,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:58,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975358964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:58,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:58,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:59,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/d451237c443c4238909bba1becdbb286 2024-11-19T00:14:59,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/1b401825aad64b7a8f078109a2d5d6b5 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/1b401825aad64b7a8f078109a2d5d6b5 2024-11-19T00:14:59,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/1b401825aad64b7a8f078109a2d5d6b5, entries=200, sequenceid=60, filesize=38.6 K 2024-11-19T00:14:59,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/1585c43935ae4b75ba3e05e01e614e94 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/1585c43935ae4b75ba3e05e01e614e94 2024-11-19T00:14:59,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/1585c43935ae4b75ba3e05e01e614e94, entries=150, sequenceid=60, filesize=11.7 K 2024-11-19T00:14:59,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/d451237c443c4238909bba1becdbb286 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/d451237c443c4238909bba1becdbb286 2024-11-19T00:14:59,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/d451237c443c4238909bba1becdbb286, entries=150, sequenceid=60, filesize=11.7 K 2024-11-19T00:14:59,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.71 KB/132820 for 999ebf86880fba150650b0d98c4af3c3 in 978ms, sequenceid=60, compaction requested=true 2024-11-19T00:14:59,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:14:59,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 999ebf86880fba150650b0d98c4af3c3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:14:59,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:59,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 999ebf86880fba150650b0d98c4af3c3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:14:59,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T00:14:59,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 999ebf86880fba150650b0d98c4af3c3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:14:59,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-19T00:14:59,109 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:59,109 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:59,115 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:59,115 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 999ebf86880fba150650b0d98c4af3c3/C is initiating minor compaction (all files) 2024-11-19T00:14:59,115 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 999ebf86880fba150650b0d98c4af3c3/C in TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:59,115 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/ba260e839d2c4bbc8717ef9bd54d4fd6, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/fccf6507d2804855b4511c76ad009113, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/d451237c443c4238909bba1becdbb286] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp, totalSize=35.2 K 2024-11-19T00:14:59,116 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:59,116 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 999ebf86880fba150650b0d98c4af3c3/A is initiating minor compaction (all files) 2024-11-19T00:14:59,116 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 999ebf86880fba150650b0d98c4af3c3/A in TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:59,116 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/ef00bb5e8f284f3a8c8e2db827b39993, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/1d562e9d38964d0fa7b4e547d7aaa824, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/1b401825aad64b7a8f078109a2d5d6b5] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp, totalSize=107.5 K 2024-11-19T00:14:59,116 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:59,116 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. files: [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/ef00bb5e8f284f3a8c8e2db827b39993, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/1d562e9d38964d0fa7b4e547d7aaa824, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/1b401825aad64b7a8f078109a2d5d6b5] 2024-11-19T00:14:59,117 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef00bb5e8f284f3a8c8e2db827b39993, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731975295682 2024-11-19T00:14:59,117 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting ba260e839d2c4bbc8717ef9bd54d4fd6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731975295682 2024-11-19T00:14:59,118 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d562e9d38964d0fa7b4e547d7aaa824, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731975295754 2024-11-19T00:14:59,118 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting fccf6507d2804855b4511c76ad009113, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731975295754 2024-11-19T00:14:59,118 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b401825aad64b7a8f078109a2d5d6b5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731975297975 2024-11-19T00:14:59,118 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting d451237c443c4238909bba1becdbb286, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731975297975 2024-11-19T00:14:59,136 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 999ebf86880fba150650b0d98c4af3c3#C#compaction#297 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:59,136 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/c25053a2def74d0eafd397931d216318 is 50, key is test_row_0/C:col10/1731975298004/Put/seqid=0 2024-11-19T00:14:59,142 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:14:59,158 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119fc456e5dbb144b949f5e59de4c5417b2_999ebf86880fba150650b0d98c4af3c3 store=[table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:14:59,161 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119fc456e5dbb144b949f5e59de4c5417b2_999ebf86880fba150650b0d98c4af3c3, store=[table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:14:59,161 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fc456e5dbb144b949f5e59de4c5417b2_999ebf86880fba150650b0d98c4af3c3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:14:59,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742174_1350 (size=12104) 2024-11-19T00:14:59,199 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/c25053a2def74d0eafd397931d216318 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/c25053a2def74d0eafd397931d216318 2024-11-19T00:14:59,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742175_1351 (size=4469) 2024-11-19T00:14:59,212 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 999ebf86880fba150650b0d98c4af3c3/C of 999ebf86880fba150650b0d98c4af3c3 into c25053a2def74d0eafd397931d216318(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:59,212 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:14:59,212 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., storeName=999ebf86880fba150650b0d98c4af3c3/C, priority=13, startTime=1731975299109; duration=0sec 2024-11-19T00:14:59,212 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:14:59,212 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 999ebf86880fba150650b0d98c4af3c3:C 2024-11-19T00:14:59,217 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:14:59,219 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:14:59,219 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 999ebf86880fba150650b0d98c4af3c3/B is initiating minor compaction (all files) 2024-11-19T00:14:59,219 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 999ebf86880fba150650b0d98c4af3c3/B in TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:14:59,219 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/429f2da816ab4ccd94da6e0cec887dd6, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/c9b19a9cf190495eb409f9312aab9358, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/1585c43935ae4b75ba3e05e01e614e94] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp, totalSize=35.2 K 2024-11-19T00:14:59,219 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 429f2da816ab4ccd94da6e0cec887dd6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731975295682 2024-11-19T00:14:59,220 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting c9b19a9cf190495eb409f9312aab9358, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731975295754 2024-11-19T00:14:59,220 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 1585c43935ae4b75ba3e05e01e614e94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731975297975 2024-11-19T00:14:59,228 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 999ebf86880fba150650b0d98c4af3c3#B#compaction#299 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:59,228 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/d5d0d142e9454670b35fa46f6ae38422 is 50, key is test_row_0/B:col10/1731975298004/Put/seqid=0 2024-11-19T00:14:59,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742176_1352 (size=12104) 2024-11-19T00:14:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:59,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 999ebf86880fba150650b0d98c4af3c3 3/3 column families, dataSize=136.42 KB heapSize=380.69 KB 2024-11-19T00:14:59,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=A 2024-11-19T00:14:59,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:14:59,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=B 2024-11-19T00:14:59,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:14:59,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=C 2024-11-19T00:14:59,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:14:59,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196c20cde7d6f049bb9e77410d7851f350_999ebf86880fba150650b0d98c4af3c3 is 50, key is test_row_0/A:col10/1731975298285/Put/seqid=0 2024-11-19T00:14:59,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742177_1353 (size=17034) 2024-11-19T00:14:59,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975359496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:59,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:59,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975359501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:59,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975359503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:59,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:59,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975359507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:59,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:59,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:59,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:59,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975359507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:59,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:59,605 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 999ebf86880fba150650b0d98c4af3c3#A#compaction#298 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:14:59,606 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/27acba2b420f47adbd471256b4389690 is 175, key is test_row_0/A:col10/1731975298004/Put/seqid=0 2024-11-19T00:14:59,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742178_1354 (size=31058) 2024-11-19T00:14:59,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:59,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975359613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:59,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:59,638 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/d5d0d142e9454670b35fa46f6ae38422 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/d5d0d142e9454670b35fa46f6ae38422 2024-11-19T00:14:59,643 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 999ebf86880fba150650b0d98c4af3c3/B of 999ebf86880fba150650b0d98c4af3c3 into d5d0d142e9454670b35fa46f6ae38422(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:14:59,643 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:14:59,643 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., storeName=999ebf86880fba150650b0d98c4af3c3/B, priority=13, startTime=1731975299109; duration=0sec 2024-11-19T00:14:59,643 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:14:59,643 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 999ebf86880fba150650b0d98c4af3c3:B 2024-11-19T00:14:59,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:59,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975359634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:59,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975359634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:59,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975359634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:59,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:59,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975359634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:59,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:59,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:59,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975359826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:59,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T00:14:59,864 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:14:59,864 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:14:59,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975359854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:59,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:59,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975359854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:59,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:59,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975359854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-19T00:14:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:14:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975359854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:14:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:14:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:14:59,867 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:14:59,868 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:14:59,868 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:14:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T00:14:59,906 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:14:59,910 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196c20cde7d6f049bb9e77410d7851f350_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196c20cde7d6f049bb9e77410d7851f350_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:14:59,911 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/77bad13e75e245cd9da2aeec36429406, store: [table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:14:59,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/77bad13e75e245cd9da2aeec36429406 is 175, key is test_row_0/A:col10/1731975298285/Put/seqid=0 2024-11-19T00:14:59,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742179_1355 (size=48139) 2024-11-19T00:14:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T00:15:00,014 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/27acba2b420f47adbd471256b4389690 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/27acba2b420f47adbd471256b4389690 2024-11-19T00:15:00,018 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 999ebf86880fba150650b0d98c4af3c3/A of 999ebf86880fba150650b0d98c4af3c3 into 27acba2b420f47adbd471256b4389690(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:00,018 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:00,018 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., storeName=999ebf86880fba150650b0d98c4af3c3/A, priority=13, startTime=1731975299109; duration=0sec 2024-11-19T00:15:00,018 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:00,018 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 999ebf86880fba150650b0d98c4af3c3:A 2024-11-19T00:15:00,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T00:15:00,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:00,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:00,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:00,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:00,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:00,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975360144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:00,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:00,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:00,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:00,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T00:15:00,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:00,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:00,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:00,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:00,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:00,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:00,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975360173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:00,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975360173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:00,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975360174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:00,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975360174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:00,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:00,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:00,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:00,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:00,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:00,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:00,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T00:15:00,315 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/77bad13e75e245cd9da2aeec36429406 2024-11-19T00:15:00,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/e0b3ef8d81e14f1cbcda495e6cd81a98 is 50, key is test_row_0/B:col10/1731975298285/Put/seqid=0 2024-11-19T00:15:00,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T00:15:00,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:00,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:00,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:00,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:00,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:00,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:00,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742180_1356 (size=12001) 2024-11-19T00:15:00,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T00:15:00,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:00,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:00,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:00,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:00,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:00,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T00:15:00,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T00:15:00,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:00,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:00,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:00,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:00,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:00,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975360663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:00,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:00,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:00,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:00,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975360684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:00,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:00,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:00,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:00,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975360694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:00,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:00,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:00,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975360694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:00,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:00,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975360694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:00,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:00,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:00,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:00,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:00,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:00,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:00,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/e0b3ef8d81e14f1cbcda495e6cd81a98 2024-11-19T00:15:00,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/2174cb09b6404526995f006fe67b1252 is 50, key is test_row_0/C:col10/1731975298285/Put/seqid=0 2024-11-19T00:15:00,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742181_1357 (size=12001) 2024-11-19T00:15:00,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T00:15:00,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:00,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:00,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:00,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:00,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:00,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T00:15:00,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:00,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:00,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:00,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:00,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:00,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T00:15:01,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T00:15:01,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:01,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:01,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:01,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:01,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:01,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:01,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/2174cb09b6404526995f006fe67b1252 2024-11-19T00:15:01,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/77bad13e75e245cd9da2aeec36429406 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/77bad13e75e245cd9da2aeec36429406 2024-11-19T00:15:01,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/77bad13e75e245cd9da2aeec36429406, entries=250, sequenceid=97, filesize=47.0 K 2024-11-19T00:15:01,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/e0b3ef8d81e14f1cbcda495e6cd81a98 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/e0b3ef8d81e14f1cbcda495e6cd81a98 2024-11-19T00:15:01,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/e0b3ef8d81e14f1cbcda495e6cd81a98, entries=150, sequenceid=97, filesize=11.7 K 2024-11-19T00:15:01,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/2174cb09b6404526995f006fe67b1252 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/2174cb09b6404526995f006fe67b1252 2024-11-19T00:15:01,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/2174cb09b6404526995f006fe67b1252, entries=150, sequenceid=97, filesize=11.7 K 2024-11-19T00:15:01,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~143.13 KB/146560, heapSize ~400.56 KB/410176, currentSize=46.96 KB/48090 for 999ebf86880fba150650b0d98c4af3c3 in 1692ms, sequenceid=97, compaction requested=false 2024-11-19T00:15:01,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:01,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T00:15:01,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:01,264 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 999ebf86880fba150650b0d98c4af3c3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T00:15:01,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=A 2024-11-19T00:15:01,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:01,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=B 2024-11-19T00:15:01,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:01,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=C 2024-11-19T00:15:01,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:01,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fdc04cd36a0a430f9d90efe15f369097_999ebf86880fba150650b0d98c4af3c3 is 50, key is test_row_0/A:col10/1731975299503/Put/seqid=0 2024-11-19T00:15:01,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742182_1358 (size=12154) 2024-11-19T00:15:01,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:01,278 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fdc04cd36a0a430f9d90efe15f369097_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fdc04cd36a0a430f9d90efe15f369097_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:01,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/7e8d97a4ea644ba79c1848ce9431521e, store: [table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:01,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/7e8d97a4ea644ba79c1848ce9431521e is 175, key is test_row_0/A:col10/1731975299503/Put/seqid=0 2024-11-19T00:15:01,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742183_1359 (size=30955) 2024-11-19T00:15:01,684 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=109, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/7e8d97a4ea644ba79c1848ce9431521e 2024-11-19T00:15:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:01,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:01,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/d21593f8e9c24b45b1dc900ba1a4b255 is 50, key is test_row_0/B:col10/1731975299503/Put/seqid=0 2024-11-19T00:15:01,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742184_1360 (size=12001) 2024-11-19T00:15:01,726 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/d21593f8e9c24b45b1dc900ba1a4b255 2024-11-19T00:15:01,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/d4c9f96447b64954afc715ae844c2dfe is 50, key is test_row_0/C:col10/1731975299503/Put/seqid=0 2024-11-19T00:15:01,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742185_1361 (size=12001) 2024-11-19T00:15:01,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731975301724, max timestamp=1731975301754Num uniques 859; 2024-11-19T00:15:01,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T00:15:01,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731975301724, max timestamp=1731975301754Num uniques 859; 2024-11-19T00:15:01,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T00:15:01,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731975301724, max timestamp=1731975301754Num uniques 859; 2024-11-19T00:15:01,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T00:15:01,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:01,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975361780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:01,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:01,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975361787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:01,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:01,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975361789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:01,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975361794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:01,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:01,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:01,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:01,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:01,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:01,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:01,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975361795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:01,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:01,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:01,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975361904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:01,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975361904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:01,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:01,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975361914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:01,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:01,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975361915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:01,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:01,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:01,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975361915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T00:15:02,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975362123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975362124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:02,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975362134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:02,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975362134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:02,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:02,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975362134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:02,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:02,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:02,145 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/d4c9f96447b64954afc715ae844c2dfe 2024-11-19T00:15:02,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/7e8d97a4ea644ba79c1848ce9431521e as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/7e8d97a4ea644ba79c1848ce9431521e 2024-11-19T00:15:02,159 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/7e8d97a4ea644ba79c1848ce9431521e, entries=150, sequenceid=109, filesize=30.2 K 2024-11-19T00:15:02,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/d21593f8e9c24b45b1dc900ba1a4b255 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/d21593f8e9c24b45b1dc900ba1a4b255 2024-11-19T00:15:02,170 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/d21593f8e9c24b45b1dc900ba1a4b255, entries=150, sequenceid=109, filesize=11.7 K 2024-11-19T00:15:02,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/d4c9f96447b64954afc715ae844c2dfe as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/d4c9f96447b64954afc715ae844c2dfe 2024-11-19T00:15:02,181 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/d4c9f96447b64954afc715ae844c2dfe, entries=150, sequenceid=109, filesize=11.7 K 2024-11-19T00:15:02,183 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 999ebf86880fba150650b0d98c4af3c3 in 918ms, sequenceid=109, compaction requested=true 2024-11-19T00:15:02,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:02,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:02,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-19T00:15:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-19T00:15:02,191 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-19T00:15:02,191 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3160 sec 2024-11-19T00:15:02,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.3300 sec 2024-11-19T00:15:02,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:02,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 999ebf86880fba150650b0d98c4af3c3 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-11-19T00:15:02,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=A 2024-11-19T00:15:02,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:02,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=B 2024-11-19T00:15:02,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:02,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=C 2024-11-19T00:15:02,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:02,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fecc750f092e4b6c8e9e628a7c1b26a8_999ebf86880fba150650b0d98c4af3c3 is 50, key is test_row_0/A:col10/1731975301792/Put/seqid=0 2024-11-19T00:15:02,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742186_1362 (size=14794) 2024-11-19T00:15:02,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:02,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975362470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:02,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:02,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:02,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975362471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:02,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:02,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:02,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975362472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:02,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:02,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975362473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:02,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:02,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:02,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:02,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975362474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:02,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:02,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975362584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:02,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975362593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:02,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975362594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:02,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:02,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975362594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:02,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:02,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:02,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975362594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:02,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:02,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:02,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975362804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:02,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975362804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:02,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:02,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975362804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:02,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975362804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:02,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975362804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:02,867 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:02,872 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fecc750f092e4b6c8e9e628a7c1b26a8_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fecc750f092e4b6c8e9e628a7c1b26a8_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:02,873 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/3362e338836c4d2588f4d7a06a021345, store: [table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:02,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/3362e338836c4d2588f4d7a06a021345 is 175, key is test_row_0/A:col10/1731975301792/Put/seqid=0 2024-11-19T00:15:02,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742187_1363 (size=39749) 2024-11-19T00:15:02,886 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=144, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/3362e338836c4d2588f4d7a06a021345 2024-11-19T00:15:02,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/9685057c5fd2466daf7895b8c36ea124 is 50, key is test_row_0/B:col10/1731975301792/Put/seqid=0 2024-11-19T00:15:02,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742188_1364 (size=12151) 2024-11-19T00:15:02,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/9685057c5fd2466daf7895b8c36ea124 2024-11-19T00:15:02,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/435831515a5b44279cddb5b6b2c84969 is 50, key is test_row_0/C:col10/1731975301792/Put/seqid=0 2024-11-19T00:15:02,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742189_1365 (size=12151) 2024-11-19T00:15:03,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:03,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975363123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:03,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:03,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975363123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975363124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:03,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975363124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:03,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975363134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:03,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:03,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/435831515a5b44279cddb5b6b2c84969 2024-11-19T00:15:03,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/3362e338836c4d2588f4d7a06a021345 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/3362e338836c4d2588f4d7a06a021345 2024-11-19T00:15:03,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/3362e338836c4d2588f4d7a06a021345, entries=200, sequenceid=144, filesize=38.8 K 2024-11-19T00:15:03,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/9685057c5fd2466daf7895b8c36ea124 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/9685057c5fd2466daf7895b8c36ea124 2024-11-19T00:15:03,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/9685057c5fd2466daf7895b8c36ea124, entries=150, sequenceid=144, filesize=11.9 K 2024-11-19T00:15:03,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/435831515a5b44279cddb5b6b2c84969 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/435831515a5b44279cddb5b6b2c84969 2024-11-19T00:15:03,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/435831515a5b44279cddb5b6b2c84969, entries=150, sequenceid=144, filesize=11.9 K 2024-11-19T00:15:03,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~406.03 KB/415776, currentSize=40.25 KB/41220 for 999ebf86880fba150650b0d98c4af3c3 in 905ms, sequenceid=144, compaction requested=true 2024-11-19T00:15:03,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:03,358 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:15:03,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 999ebf86880fba150650b0d98c4af3c3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:15:03,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:03,359 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:15:03,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 999ebf86880fba150650b0d98c4af3c3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:15:03,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:03,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 999ebf86880fba150650b0d98c4af3c3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:15:03,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:03,359 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149901 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:15:03,359 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 999ebf86880fba150650b0d98c4af3c3/A is initiating minor compaction (all files) 2024-11-19T00:15:03,359 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 999ebf86880fba150650b0d98c4af3c3/A in TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:03,360 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/27acba2b420f47adbd471256b4389690, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/77bad13e75e245cd9da2aeec36429406, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/7e8d97a4ea644ba79c1848ce9431521e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/3362e338836c4d2588f4d7a06a021345] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp, totalSize=146.4 K 2024-11-19T00:15:03,360 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:03,360 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. files: [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/27acba2b420f47adbd471256b4389690, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/77bad13e75e245cd9da2aeec36429406, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/7e8d97a4ea644ba79c1848ce9431521e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/3362e338836c4d2588f4d7a06a021345] 2024-11-19T00:15:03,360 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:15:03,360 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27acba2b420f47adbd471256b4389690, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731975297975 2024-11-19T00:15:03,360 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 999ebf86880fba150650b0d98c4af3c3/B is initiating minor compaction (all files) 2024-11-19T00:15:03,360 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 999ebf86880fba150650b0d98c4af3c3/B in TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:03,361 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/d5d0d142e9454670b35fa46f6ae38422, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/e0b3ef8d81e14f1cbcda495e6cd81a98, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/d21593f8e9c24b45b1dc900ba1a4b255, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/9685057c5fd2466daf7895b8c36ea124] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp, totalSize=47.1 K 2024-11-19T00:15:03,361 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77bad13e75e245cd9da2aeec36429406, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731975298279 2024-11-19T00:15:03,361 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting d5d0d142e9454670b35fa46f6ae38422, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731975297975 2024-11-19T00:15:03,361 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e8d97a4ea644ba79c1848ce9431521e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731975299485 2024-11-19T00:15:03,361 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3362e338836c4d2588f4d7a06a021345, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731975301787 2024-11-19T00:15:03,361 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting e0b3ef8d81e14f1cbcda495e6cd81a98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731975298285 2024-11-19T00:15:03,362 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting d21593f8e9c24b45b1dc900ba1a4b255, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731975299485 2024-11-19T00:15:03,362 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 9685057c5fd2466daf7895b8c36ea124, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731975301792 2024-11-19T00:15:03,381 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 999ebf86880fba150650b0d98c4af3c3#B#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:03,381 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/d0bb80a5b0f94030b3bcbebf0cefc616 is 50, key is test_row_0/B:col10/1731975301792/Put/seqid=0 2024-11-19T00:15:03,386 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:03,406 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111946ee59648a5e40e39ea64c8a6657b772_999ebf86880fba150650b0d98c4af3c3 store=[table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:03,409 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111946ee59648a5e40e39ea64c8a6657b772_999ebf86880fba150650b0d98c4af3c3, store=[table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:03,409 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111946ee59648a5e40e39ea64c8a6657b772_999ebf86880fba150650b0d98c4af3c3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:03,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742190_1366 (size=12391) 2024-11-19T00:15:03,422 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/d0bb80a5b0f94030b3bcbebf0cefc616 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/d0bb80a5b0f94030b3bcbebf0cefc616 2024-11-19T00:15:03,430 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 999ebf86880fba150650b0d98c4af3c3/B of 999ebf86880fba150650b0d98c4af3c3 into d0bb80a5b0f94030b3bcbebf0cefc616(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:03,430 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:03,430 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., storeName=999ebf86880fba150650b0d98c4af3c3/B, priority=12, startTime=1731975303359; duration=0sec 2024-11-19T00:15:03,430 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:03,430 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 999ebf86880fba150650b0d98c4af3c3:B 2024-11-19T00:15:03,431 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:15:03,433 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:15:03,433 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 999ebf86880fba150650b0d98c4af3c3/C is initiating minor compaction (all files) 2024-11-19T00:15:03,433 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 999ebf86880fba150650b0d98c4af3c3/C in TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:03,433 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/c25053a2def74d0eafd397931d216318, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/2174cb09b6404526995f006fe67b1252, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/d4c9f96447b64954afc715ae844c2dfe, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/435831515a5b44279cddb5b6b2c84969] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp, totalSize=47.1 K 2024-11-19T00:15:03,433 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting c25053a2def74d0eafd397931d216318, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731975297975 2024-11-19T00:15:03,434 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 2174cb09b6404526995f006fe67b1252, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731975298285 2024-11-19T00:15:03,434 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting d4c9f96447b64954afc715ae844c2dfe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731975299485 2024-11-19T00:15:03,435 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 435831515a5b44279cddb5b6b2c84969, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731975301792 2024-11-19T00:15:03,445 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 999ebf86880fba150650b0d98c4af3c3#C#compaction#311 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:03,446 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/f914d01b537a4b2883264841f6f1074e is 50, key is test_row_0/C:col10/1731975301792/Put/seqid=0 2024-11-19T00:15:03,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742191_1367 (size=4469) 2024-11-19T00:15:03,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742192_1368 (size=12391) 2024-11-19T00:15:03,478 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/f914d01b537a4b2883264841f6f1074e as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/f914d01b537a4b2883264841f6f1074e 2024-11-19T00:15:03,487 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 999ebf86880fba150650b0d98c4af3c3/C of 999ebf86880fba150650b0d98c4af3c3 into f914d01b537a4b2883264841f6f1074e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:03,488 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:03,488 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., storeName=999ebf86880fba150650b0d98c4af3c3/C, priority=12, startTime=1731975303359; duration=0sec 2024-11-19T00:15:03,488 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:03,488 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 999ebf86880fba150650b0d98c4af3c3:C 2024-11-19T00:15:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:03,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 999ebf86880fba150650b0d98c4af3c3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T00:15:03,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=A 2024-11-19T00:15:03,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:03,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=B 2024-11-19T00:15:03,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:03,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=C 2024-11-19T00:15:03,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:03,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111914345c3f9de2421982c904a55b1c46df_999ebf86880fba150650b0d98c4af3c3 is 50, key is test_row_0/A:col10/1731975302454/Put/seqid=0 2024-11-19T00:15:03,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742193_1369 (size=12304) 2024-11-19T00:15:03,699 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:03,705 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111914345c3f9de2421982c904a55b1c46df_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111914345c3f9de2421982c904a55b1c46df_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:03,706 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/3aa8ea01b5d34f64b9b24af45f84804f, store: [table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:03,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/3aa8ea01b5d34f64b9b24af45f84804f is 175, key is test_row_0/A:col10/1731975302454/Put/seqid=0 2024-11-19T00:15:03,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975363696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:03,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975363708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:03,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:03,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:03,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975363710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:03,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:03,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:03,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975363711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:03,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975363714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:03,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742194_1370 (size=31105) 2024-11-19T00:15:03,752 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/3aa8ea01b5d34f64b9b24af45f84804f 2024-11-19T00:15:03,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/29c68ac57bc0460482b1bd434c41943b is 50, key is test_row_0/B:col10/1731975302454/Put/seqid=0 2024-11-19T00:15:03,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742195_1371 (size=12151) 2024-11-19T00:15:03,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975363824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:03,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:03,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975363824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:03,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:03,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:03,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975363834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:03,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975363835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:03,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:03,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:03,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975363835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:03,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:03,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:03,853 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 999ebf86880fba150650b0d98c4af3c3#A#compaction#310 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:03,854 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/b6fe8cd07c454839a7135199b7a8aff9 is 175, key is test_row_0/A:col10/1731975301792/Put/seqid=0 2024-11-19T00:15:03,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742196_1372 (size=31345) 2024-11-19T00:15:03,866 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/b6fe8cd07c454839a7135199b7a8aff9 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/b6fe8cd07c454839a7135199b7a8aff9 2024-11-19T00:15:03,871 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 999ebf86880fba150650b0d98c4af3c3/A of 999ebf86880fba150650b0d98c4af3c3 into b6fe8cd07c454839a7135199b7a8aff9(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:03,871 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:03,871 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., storeName=999ebf86880fba150650b0d98c4af3c3/A, priority=12, startTime=1731975303358; duration=0sec 2024-11-19T00:15:03,871 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:03,871 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 999ebf86880fba150650b0d98c4af3c3:A 2024-11-19T00:15:04,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T00:15:04,024 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:15:04,024 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:15:04,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-19T00:15:04,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T00:15:04,027 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:15:04,027 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:15:04,027 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:15:04,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:04,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975364044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:04,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:04,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975364044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:04,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:04,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:04,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975364044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:04,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:04,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:04,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975364044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:04,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:04,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:04,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:04,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975364045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:04,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:04,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T00:15:04,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/29c68ac57bc0460482b1bd434c41943b 2024-11-19T00:15:04,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/1719ec3be94048dfb94834bb6b0e280e is 50, key is test_row_0/C:col10/1731975302454/Put/seqid=0 2024-11-19T00:15:04,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T00:15:04,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:04,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:04,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:04,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:04,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:04,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742197_1373 (size=12151) 2024-11-19T00:15:04,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T00:15:04,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:04,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:04,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:04,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:04,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:04,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T00:15:04,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:04,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975364354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:04,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:04,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975364354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:04,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:04,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:04,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975364354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:04,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:04,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975364354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:04,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975364354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:04,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T00:15:04,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:04,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:04,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:04,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:04,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:04,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:04,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/1719ec3be94048dfb94834bb6b0e280e 2024-11-19T00:15:04,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/3aa8ea01b5d34f64b9b24af45f84804f as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/3aa8ea01b5d34f64b9b24af45f84804f 2024-11-19T00:15:04,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/3aa8ea01b5d34f64b9b24af45f84804f, entries=150, sequenceid=159, filesize=30.4 K 2024-11-19T00:15:04,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/29c68ac57bc0460482b1bd434c41943b as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/29c68ac57bc0460482b1bd434c41943b 2024-11-19T00:15:04,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/29c68ac57bc0460482b1bd434c41943b, entries=150, sequenceid=159, filesize=11.9 K 2024-11-19T00:15:04,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/1719ec3be94048dfb94834bb6b0e280e as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/1719ec3be94048dfb94834bb6b0e280e 2024-11-19T00:15:04,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/1719ec3be94048dfb94834bb6b0e280e, entries=150, sequenceid=159, filesize=11.9 K 2024-11-19T00:15:04,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 999ebf86880fba150650b0d98c4af3c3 in 953ms, sequenceid=159, compaction requested=false 2024-11-19T00:15:04,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:04,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T00:15:04,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:04,643 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 999ebf86880fba150650b0d98c4af3c3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T00:15:04,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=A 2024-11-19T00:15:04,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:04,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=B 2024-11-19T00:15:04,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:04,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=C 2024-11-19T00:15:04,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:04,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T00:15:04,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c88926d45cbe4451ab546b60481cb977_999ebf86880fba150650b0d98c4af3c3 is 50, key is test_row_0/A:col10/1731975303712/Put/seqid=0 2024-11-19T00:15:04,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742198_1374 (size=12304) 2024-11-19T00:15:04,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:04,685 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c88926d45cbe4451ab546b60481cb977_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c88926d45cbe4451ab546b60481cb977_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:04,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/96bb017918f5458e97913432f4270a08, store: [table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:04,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/96bb017918f5458e97913432f4270a08 is 175, key is test_row_0/A:col10/1731975303712/Put/seqid=0 2024-11-19T00:15:04,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742199_1375 (size=31105) 2024-11-19T00:15:04,698 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=179, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/96bb017918f5458e97913432f4270a08 2024-11-19T00:15:04,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/0e96da03a4a94c0da0a264e7bfbee2a2 is 50, key is test_row_0/B:col10/1731975303712/Put/seqid=0 2024-11-19T00:15:04,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742200_1376 (size=12151) 2024-11-19T00:15:04,737 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/0e96da03a4a94c0da0a264e7bfbee2a2 2024-11-19T00:15:04,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/7332efe7d2f04e61a7cd3403d3917ed7 is 50, key is test_row_0/C:col10/1731975303712/Put/seqid=0 2024-11-19T00:15:04,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742201_1377 (size=12151) 2024-11-19T00:15:04,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:04,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:04,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:04,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975364882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:04,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:04,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:04,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975364883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:04,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:04,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:04,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975364884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:04,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:04,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975364885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:04,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:04,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:04,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:04,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975364885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:04,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:04,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975364994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:05,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:05,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975365004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:05,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975365004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:05,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975365004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:05,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975365004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T00:15:05,175 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/7332efe7d2f04e61a7cd3403d3917ed7 2024-11-19T00:15:05,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/96bb017918f5458e97913432f4270a08 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/96bb017918f5458e97913432f4270a08 2024-11-19T00:15:05,185 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/96bb017918f5458e97913432f4270a08, entries=150, sequenceid=179, filesize=30.4 K 2024-11-19T00:15:05,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/0e96da03a4a94c0da0a264e7bfbee2a2 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/0e96da03a4a94c0da0a264e7bfbee2a2 2024-11-19T00:15:05,192 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/0e96da03a4a94c0da0a264e7bfbee2a2, entries=150, sequenceid=179, filesize=11.9 K 2024-11-19T00:15:05,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/7332efe7d2f04e61a7cd3403d3917ed7 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/7332efe7d2f04e61a7cd3403d3917ed7 2024-11-19T00:15:05,196 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/7332efe7d2f04e61a7cd3403d3917ed7, entries=150, sequenceid=179, filesize=11.9 K 2024-11-19T00:15:05,197 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 999ebf86880fba150650b0d98c4af3c3 in 553ms, sequenceid=179, compaction requested=true 2024-11-19T00:15:05,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:05,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:05,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-19T00:15:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-19T00:15:05,199 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-19T00:15:05,199 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1710 sec 2024-11-19T00:15:05,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.1760 sec 2024-11-19T00:15:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:05,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 999ebf86880fba150650b0d98c4af3c3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T00:15:05,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=A 2024-11-19T00:15:05,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:05,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=B 2024-11-19T00:15:05,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:05,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=C 2024-11-19T00:15:05,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:05,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e75292aad4874986bfda59a392dedd1d_999ebf86880fba150650b0d98c4af3c3 is 50, key is test_row_0/A:col10/1731975304868/Put/seqid=0 2024-11-19T00:15:05,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742202_1378 (size=14794) 2024-11-19T00:15:05,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975365242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:05,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:05,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975365242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:05,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:05,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:05,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:05,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975365256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:05,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975365255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:05,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:05,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:05,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975365259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:05,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:05,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975365363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:05,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:05,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975365363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:05,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:05,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:05,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975365384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:05,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:05,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975365384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:05,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:05,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975365384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:05,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:05,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975365584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975365584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:05,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975365603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:05,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:05,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975365603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:05,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:05,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975365603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:05,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:05,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:05,627 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:05,631 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e75292aad4874986bfda59a392dedd1d_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e75292aad4874986bfda59a392dedd1d_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:05,632 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/c59aae861f0d4fe58bc7ad5171522a3e, store: [table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:05,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/c59aae861f0d4fe58bc7ad5171522a3e is 175, key is test_row_0/A:col10/1731975304868/Put/seqid=0 2024-11-19T00:15:05,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742203_1379 (size=39749) 2024-11-19T00:15:05,643 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/c59aae861f0d4fe58bc7ad5171522a3e 2024-11-19T00:15:05,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/52577767d86747d1acb62a9c8784559c is 50, key is test_row_0/B:col10/1731975304868/Put/seqid=0 2024-11-19T00:15:05,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742204_1380 (size=12151) 2024-11-19T00:15:05,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/52577767d86747d1acb62a9c8784559c 2024-11-19T00:15:05,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/424e63ba321d4a2aaca18ea951888730 is 50, key is test_row_0/C:col10/1731975304868/Put/seqid=0 2024-11-19T00:15:05,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742205_1381 (size=12151) 2024-11-19T00:15:05,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975365903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:05,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:05,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975365903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:05,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:05,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975365924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:05,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975365924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:05,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:05,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975365924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:05,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:05,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:05,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:06,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/424e63ba321d4a2aaca18ea951888730 2024-11-19T00:15:06,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/c59aae861f0d4fe58bc7ad5171522a3e as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/c59aae861f0d4fe58bc7ad5171522a3e 2024-11-19T00:15:06,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/c59aae861f0d4fe58bc7ad5171522a3e, entries=200, sequenceid=194, filesize=38.8 K 2024-11-19T00:15:06,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/52577767d86747d1acb62a9c8784559c as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/52577767d86747d1acb62a9c8784559c 2024-11-19T00:15:06,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/52577767d86747d1acb62a9c8784559c, entries=150, sequenceid=194, filesize=11.9 K 2024-11-19T00:15:06,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/424e63ba321d4a2aaca18ea951888730 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/424e63ba321d4a2aaca18ea951888730 2024-11-19T00:15:06,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/424e63ba321d4a2aaca18ea951888730, entries=150, sequenceid=194, filesize=11.9 K 2024-11-19T00:15:06,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 999ebf86880fba150650b0d98c4af3c3 in 903ms, sequenceid=194, compaction requested=true 2024-11-19T00:15:06,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:06,114 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:15:06,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 999ebf86880fba150650b0d98c4af3c3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:15:06,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:06,114 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:15:06,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 999ebf86880fba150650b0d98c4af3c3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:15:06,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:06,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 999ebf86880fba150650b0d98c4af3c3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:15:06,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:06,117 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133304 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:15:06,118 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 999ebf86880fba150650b0d98c4af3c3/A is initiating minor compaction (all files) 2024-11-19T00:15:06,118 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 999ebf86880fba150650b0d98c4af3c3/A in TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:06,118 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/b6fe8cd07c454839a7135199b7a8aff9, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/3aa8ea01b5d34f64b9b24af45f84804f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/96bb017918f5458e97913432f4270a08, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/c59aae861f0d4fe58bc7ad5171522a3e] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp, totalSize=130.2 K 2024-11-19T00:15:06,118 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:06,118 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. files: [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/b6fe8cd07c454839a7135199b7a8aff9, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/3aa8ea01b5d34f64b9b24af45f84804f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/96bb017918f5458e97913432f4270a08, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/c59aae861f0d4fe58bc7ad5171522a3e] 2024-11-19T00:15:06,118 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:15:06,118 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 999ebf86880fba150650b0d98c4af3c3/B is initiating minor compaction (all files) 2024-11-19T00:15:06,118 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 999ebf86880fba150650b0d98c4af3c3/B in TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:06,119 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/d0bb80a5b0f94030b3bcbebf0cefc616, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/29c68ac57bc0460482b1bd434c41943b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/0e96da03a4a94c0da0a264e7bfbee2a2, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/52577767d86747d1acb62a9c8784559c] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp, totalSize=47.7 K 2024-11-19T00:15:06,119 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6fe8cd07c454839a7135199b7a8aff9, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731975301792 2024-11-19T00:15:06,119 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting d0bb80a5b0f94030b3bcbebf0cefc616, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731975301792 2024-11-19T00:15:06,119 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3aa8ea01b5d34f64b9b24af45f84804f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731975302454 2024-11-19T00:15:06,120 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 29c68ac57bc0460482b1bd434c41943b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731975302454 2024-11-19T00:15:06,120 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96bb017918f5458e97913432f4270a08, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731975303689 2024-11-19T00:15:06,120 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e96da03a4a94c0da0a264e7bfbee2a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731975303689 2024-11-19T00:15:06,120 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting c59aae861f0d4fe58bc7ad5171522a3e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731975304868 2024-11-19T00:15:06,121 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 52577767d86747d1acb62a9c8784559c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731975304868 2024-11-19T00:15:06,126 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:06,128 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411199f576b0ebd7c4b0384f04504cb7387c7_999ebf86880fba150650b0d98c4af3c3 store=[table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:06,128 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 999ebf86880fba150650b0d98c4af3c3#B#compaction#322 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:06,129 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/d8d493eacfc34810ac87a514949e8edb is 50, key is test_row_0/B:col10/1731975304868/Put/seqid=0 2024-11-19T00:15:06,131 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411199f576b0ebd7c4b0384f04504cb7387c7_999ebf86880fba150650b0d98c4af3c3, store=[table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:06,131 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199f576b0ebd7c4b0384f04504cb7387c7_999ebf86880fba150650b0d98c4af3c3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:06,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742207_1383 (size=4469) 2024-11-19T00:15:06,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742206_1382 (size=12527) 2024-11-19T00:15:06,146 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 999ebf86880fba150650b0d98c4af3c3#A#compaction#321 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:06,146 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/d818d93551e74f17a2935e785de8f95a is 175, key is test_row_0/A:col10/1731975304868/Put/seqid=0 2024-11-19T00:15:06,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742208_1384 (size=31481) 2024-11-19T00:15:06,162 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/d818d93551e74f17a2935e785de8f95a as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/d818d93551e74f17a2935e785de8f95a 2024-11-19T00:15:06,166 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 999ebf86880fba150650b0d98c4af3c3/A of 999ebf86880fba150650b0d98c4af3c3 into d818d93551e74f17a2935e785de8f95a(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:06,166 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:06,166 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., storeName=999ebf86880fba150650b0d98c4af3c3/A, priority=12, startTime=1731975306114; duration=0sec 2024-11-19T00:15:06,166 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:06,166 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 999ebf86880fba150650b0d98c4af3c3:A 2024-11-19T00:15:06,166 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:15:06,168 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:15:06,168 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 999ebf86880fba150650b0d98c4af3c3/C is initiating minor compaction (all files) 2024-11-19T00:15:06,168 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 999ebf86880fba150650b0d98c4af3c3/C in TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:06,168 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/f914d01b537a4b2883264841f6f1074e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/1719ec3be94048dfb94834bb6b0e280e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/7332efe7d2f04e61a7cd3403d3917ed7, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/424e63ba321d4a2aaca18ea951888730] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp, totalSize=47.7 K 2024-11-19T00:15:06,170 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting f914d01b537a4b2883264841f6f1074e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731975301792 2024-11-19T00:15:06,170 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1719ec3be94048dfb94834bb6b0e280e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731975302454 2024-11-19T00:15:06,170 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7332efe7d2f04e61a7cd3403d3917ed7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731975303689 2024-11-19T00:15:06,171 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 424e63ba321d4a2aaca18ea951888730, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731975304868 2024-11-19T00:15:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T00:15:06,174 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:15:06,175 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:15:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-19T00:15:06,178 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 999ebf86880fba150650b0d98c4af3c3#C#compaction#323 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T00:15:06,178 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:15:06,179 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/4045ff10b4cc4757bcdb9696df63a33c is 50, key is test_row_0/C:col10/1731975304868/Put/seqid=0 2024-11-19T00:15:06,182 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:15:06,183 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:15:06,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742209_1385 (size=12527) 2024-11-19T00:15:06,218 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/4045ff10b4cc4757bcdb9696df63a33c as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/4045ff10b4cc4757bcdb9696df63a33c 2024-11-19T00:15:06,225 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 999ebf86880fba150650b0d98c4af3c3/C of 999ebf86880fba150650b0d98c4af3c3 into 4045ff10b4cc4757bcdb9696df63a33c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:06,225 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:06,225 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., storeName=999ebf86880fba150650b0d98c4af3c3/C, priority=12, startTime=1731975306114; duration=0sec 2024-11-19T00:15:06,225 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:06,225 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 999ebf86880fba150650b0d98c4af3c3:C 2024-11-19T00:15:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T00:15:06,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T00:15:06,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:06,337 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 999ebf86880fba150650b0d98c4af3c3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T00:15:06,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=A 2024-11-19T00:15:06,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:06,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=B 2024-11-19T00:15:06,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:06,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=C 2024-11-19T00:15:06,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:06,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197fb229afffdb4cc283598e100a1b76b7_999ebf86880fba150650b0d98c4af3c3 is 50, key is test_row_0/A:col10/1731975305253/Put/seqid=0 2024-11-19T00:15:06,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742210_1386 (size=12304) 2024-11-19T00:15:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:06,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:06,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975366450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:06,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:06,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975366455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:06,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:06,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975366458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:06,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:06,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975366459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:06,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975366464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T00:15:06,555 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/d8d493eacfc34810ac87a514949e8edb as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/d8d493eacfc34810ac87a514949e8edb 2024-11-19T00:15:06,563 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 999ebf86880fba150650b0d98c4af3c3/B of 999ebf86880fba150650b0d98c4af3c3 into d8d493eacfc34810ac87a514949e8edb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:06,563 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:06,563 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., storeName=999ebf86880fba150650b0d98c4af3c3/B, priority=12, startTime=1731975306114; duration=0sec 2024-11-19T00:15:06,563 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:06,563 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 999ebf86880fba150650b0d98c4af3c3:B 2024-11-19T00:15:06,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975366574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:06,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:06,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975366574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:06,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:06,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975366574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:06,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:06,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975366574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:06,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975366575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:06,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:06,761 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197fb229afffdb4cc283598e100a1b76b7_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197fb229afffdb4cc283598e100a1b76b7_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:06,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/114dfc8ae90e4dc4ac7006967a20aa3a, store: [table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:06,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/114dfc8ae90e4dc4ac7006967a20aa3a is 175, key is test_row_0/A:col10/1731975305253/Put/seqid=0 2024-11-19T00:15:06,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742211_1387 (size=31105) 2024-11-19T00:15:06,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:06,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975366794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:06,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:06,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:06,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975366795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:06,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:06,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:06,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975366795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:06,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975366795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:06,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975366795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T00:15:07,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975367104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:07,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:07,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975367104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:07,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:07,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:07,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:07,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:07,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:07,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975367104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:07,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:07,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:07,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:07,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975367114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:07,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:07,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975367114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:07,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:07,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:07,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:07,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:07,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:07,185 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/114dfc8ae90e4dc4ac7006967a20aa3a 2024-11-19T00:15:07,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/97fd6853d807438b9717db920ef8aadc is 50, key is test_row_0/B:col10/1731975305253/Put/seqid=0 2024-11-19T00:15:07,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742212_1388 (size=12151) 2024-11-19T00:15:07,201 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/97fd6853d807438b9717db920ef8aadc 2024-11-19T00:15:07,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/37834bce25b242229f2d65230e21bf40 is 50, key is test_row_0/C:col10/1731975305253/Put/seqid=0 2024-11-19T00:15:07,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742213_1389 (size=12151) 2024-11-19T00:15:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T00:15:07,620 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/37834bce25b242229f2d65230e21bf40 2024-11-19T00:15:07,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/114dfc8ae90e4dc4ac7006967a20aa3a as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/114dfc8ae90e4dc4ac7006967a20aa3a 2024-11-19T00:15:07,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:07,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975367624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:07,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:07,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:07,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:07,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:07,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975367624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:07,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:07,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975367624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:07,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:07,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:07,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:07,635 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/114dfc8ae90e4dc4ac7006967a20aa3a, entries=150, sequenceid=213, filesize=30.4 K 2024-11-19T00:15:07,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/97fd6853d807438b9717db920ef8aadc as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/97fd6853d807438b9717db920ef8aadc 2024-11-19T00:15:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:07,642 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/97fd6853d807438b9717db920ef8aadc, entries=150, sequenceid=213, filesize=11.9 K 2024-11-19T00:15:07,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975367634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:07,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/37834bce25b242229f2d65230e21bf40 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/37834bce25b242229f2d65230e21bf40 2024-11-19T00:15:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:07,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:07,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975367640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:07,651 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/37834bce25b242229f2d65230e21bf40, entries=150, sequenceid=213, filesize=11.9 K 2024-11-19T00:15:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:07,653 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 999ebf86880fba150650b0d98c4af3c3 in 1316ms, sequenceid=213, compaction requested=false 2024-11-19T00:15:07,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:07,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:07,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-19T00:15:07,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-19T00:15:07,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-19T00:15:07,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4720 sec 2024-11-19T00:15:07,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.4820 sec 2024-11-19T00:15:08,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T00:15:08,334 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:15:08,334 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:15:08,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-19T00:15:08,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T00:15:08,336 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:15:08,336 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:15:08,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:15:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T00:15:08,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T00:15:08,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:08,489 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 999ebf86880fba150650b0d98c4af3c3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T00:15:08,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=A 2024-11-19T00:15:08,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:08,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=B 2024-11-19T00:15:08,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:08,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=C 2024-11-19T00:15:08,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:08,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119954427da75bc4eedb7174a4f52ebc21b_999ebf86880fba150650b0d98c4af3c3 is 50, key is test_row_0/A:col10/1731975306448/Put/seqid=0 2024-11-19T00:15:08,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742214_1390 (size=12304) 2024-11-19T00:15:08,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:08,516 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119954427da75bc4eedb7174a4f52ebc21b_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119954427da75bc4eedb7174a4f52ebc21b_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:08,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/ed20d48bd0264adbb33d70119f51769f, store: [table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:08,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/ed20d48bd0264adbb33d70119f51769f is 175, key is test_row_0/A:col10/1731975306448/Put/seqid=0 2024-11-19T00:15:08,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742215_1391 (size=31105) 2024-11-19T00:15:08,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:08,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:08,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T00:15:08,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:08,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975368676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:08,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:08,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:08,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:08,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975368677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:08,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:08,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:08,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:08,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:08,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:08,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975368679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:08,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:08,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:08,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975368691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:08,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975368691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:08,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:08,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975368794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:08,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:08,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975368794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:08,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:08,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975368803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:08,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:08,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:08,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:08,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975368834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:08,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:08,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:08,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975368835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:08,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:08,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:08,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:08,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:08,922 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/ed20d48bd0264adbb33d70119f51769f 2024-11-19T00:15:08,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/190d3591e61447dd838c2ff6cefc2912 is 50, key is test_row_0/B:col10/1731975306448/Put/seqid=0 2024-11-19T00:15:08,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742216_1392 (size=12151) 2024-11-19T00:15:08,948 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/190d3591e61447dd838c2ff6cefc2912 2024-11-19T00:15:08,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/eb5f6d6f91eb4705a4810d78b3b110e8 is 50, key is test_row_0/C:col10/1731975306448/Put/seqid=0 2024-11-19T00:15:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T00:15:08,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742217_1393 (size=12151) 2024-11-19T00:15:08,974 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/eb5f6d6f91eb4705a4810d78b3b110e8 2024-11-19T00:15:08,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/ed20d48bd0264adbb33d70119f51769f as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/ed20d48bd0264adbb33d70119f51769f 2024-11-19T00:15:08,987 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/ed20d48bd0264adbb33d70119f51769f, entries=150, sequenceid=229, filesize=30.4 K 2024-11-19T00:15:08,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/190d3591e61447dd838c2ff6cefc2912 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/190d3591e61447dd838c2ff6cefc2912 2024-11-19T00:15:08,992 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/190d3591e61447dd838c2ff6cefc2912, entries=150, sequenceid=229, filesize=11.9 K 2024-11-19T00:15:08,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/eb5f6d6f91eb4705a4810d78b3b110e8 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/eb5f6d6f91eb4705a4810d78b3b110e8 2024-11-19T00:15:09,006 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/eb5f6d6f91eb4705a4810d78b3b110e8, entries=150, sequenceid=229, filesize=11.9 K 2024-11-19T00:15:09,007 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 999ebf86880fba150650b0d98c4af3c3 in 518ms, sequenceid=229, compaction requested=true 2024-11-19T00:15:09,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:09,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:09,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-19T00:15:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-19T00:15:09,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-19T00:15:09,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 672 msec 2024-11-19T00:15:09,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 677 msec 2024-11-19T00:15:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:09,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 999ebf86880fba150650b0d98c4af3c3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T00:15:09,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=A 2024-11-19T00:15:09,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:09,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=B 2024-11-19T00:15:09,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:09,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=C 2024-11-19T00:15:09,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:09,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192ad49cf2ad7642d9a6315e21ceaff4a7_999ebf86880fba150650b0d98c4af3c3 is 50, key is test_row_0/A:col10/1731975309019/Put/seqid=0 2024-11-19T00:15:09,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742218_1394 (size=14794) 2024-11-19T00:15:09,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:09,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975369044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:09,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:09,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975369046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:09,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975369048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:09,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:09,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975369053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:09,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:09,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975369054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:09,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:09,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975369164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:09,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975369164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:09,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:09,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:09,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975369174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:09,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:09,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975369174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:09,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:09,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:09,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:09,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975369174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:09,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:09,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975369374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:09,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:09,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975369374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:09,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:09,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975369394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:09,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:09,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975369394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:09,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:09,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975369394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:09,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:09,437 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:09,444 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192ad49cf2ad7642d9a6315e21ceaff4a7_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192ad49cf2ad7642d9a6315e21ceaff4a7_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:09,445 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/ea953303e13f4ef19f59243975f91dd5, store: [table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:09,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/ea953303e13f4ef19f59243975f91dd5 is 175, key is test_row_0/A:col10/1731975309019/Put/seqid=0 2024-11-19T00:15:09,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742219_1395 (size=39749) 2024-11-19T00:15:09,455 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/ea953303e13f4ef19f59243975f91dd5 2024-11-19T00:15:09,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/1f47f1626a624f709c6993d44ef27f11 is 50, key is test_row_0/B:col10/1731975309019/Put/seqid=0 2024-11-19T00:15:09,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742220_1396 (size=12151) 2024-11-19T00:15:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T00:15:09,473 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:15:09,474 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:15:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-19T00:15:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T00:15:09,476 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:15:09,476 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:15:09,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:15:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T00:15:09,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T00:15:09,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:09,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:09,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:09,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:09,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:09,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975369693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:09,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975369693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:09,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:09,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:09,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975369714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:09,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:09,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975369714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:09,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:09,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975369714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:09,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:09,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:09,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T00:15:09,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:09,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:09,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:09,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:09,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:09,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T00:15:09,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/1f47f1626a624f709c6993d44ef27f11 2024-11-19T00:15:09,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/4188012f0d15407dac3d2133250b4186 is 50, key is test_row_0/C:col10/1731975309019/Put/seqid=0 2024-11-19T00:15:09,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742221_1397 (size=12151) 2024-11-19T00:15:09,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/4188012f0d15407dac3d2133250b4186 2024-11-19T00:15:09,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/ea953303e13f4ef19f59243975f91dd5 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/ea953303e13f4ef19f59243975f91dd5 2024-11-19T00:15:09,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/ea953303e13f4ef19f59243975f91dd5, entries=200, sequenceid=249, filesize=38.8 K 2024-11-19T00:15:09,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/1f47f1626a624f709c6993d44ef27f11 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/1f47f1626a624f709c6993d44ef27f11 2024-11-19T00:15:09,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/1f47f1626a624f709c6993d44ef27f11, entries=150, sequenceid=249, filesize=11.9 K 2024-11-19T00:15:09,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/4188012f0d15407dac3d2133250b4186 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/4188012f0d15407dac3d2133250b4186 2024-11-19T00:15:09,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/4188012f0d15407dac3d2133250b4186, entries=150, sequenceid=249, filesize=11.9 K 2024-11-19T00:15:09,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 999ebf86880fba150650b0d98c4af3c3 in 912ms, sequenceid=249, compaction requested=true 2024-11-19T00:15:09,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:09,932 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:15:09,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 999ebf86880fba150650b0d98c4af3c3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:15:09,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:09,933 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:15:09,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 999ebf86880fba150650b0d98c4af3c3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:15:09,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:09,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 999ebf86880fba150650b0d98c4af3c3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:15:09,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:09,936 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133440 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:15:09,936 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 999ebf86880fba150650b0d98c4af3c3/A is initiating minor compaction (all files) 2024-11-19T00:15:09,936 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 999ebf86880fba150650b0d98c4af3c3/A in TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:09,936 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/d818d93551e74f17a2935e785de8f95a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/114dfc8ae90e4dc4ac7006967a20aa3a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/ed20d48bd0264adbb33d70119f51769f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/ea953303e13f4ef19f59243975f91dd5] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp, totalSize=130.3 K 2024-11-19T00:15:09,936 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:09,936 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. files: [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/d818d93551e74f17a2935e785de8f95a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/114dfc8ae90e4dc4ac7006967a20aa3a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/ed20d48bd0264adbb33d70119f51769f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/ea953303e13f4ef19f59243975f91dd5] 2024-11-19T00:15:09,939 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting d818d93551e74f17a2935e785de8f95a, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731975304868 2024-11-19T00:15:09,939 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:15:09,939 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 999ebf86880fba150650b0d98c4af3c3/B is initiating minor compaction (all files) 2024-11-19T00:15:09,939 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 999ebf86880fba150650b0d98c4af3c3/B in TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:09,939 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/d8d493eacfc34810ac87a514949e8edb, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/97fd6853d807438b9717db920ef8aadc, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/190d3591e61447dd838c2ff6cefc2912, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/1f47f1626a624f709c6993d44ef27f11] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp, totalSize=47.8 K 2024-11-19T00:15:09,939 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 114dfc8ae90e4dc4ac7006967a20aa3a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731975305239 2024-11-19T00:15:09,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T00:15:09,940 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting d8d493eacfc34810ac87a514949e8edb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731975304868 2024-11-19T00:15:09,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:09,940 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 999ebf86880fba150650b0d98c4af3c3 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T00:15:09,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=A 2024-11-19T00:15:09,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:09,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=B 2024-11-19T00:15:09,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:09,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=C 2024-11-19T00:15:09,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:09,941 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed20d48bd0264adbb33d70119f51769f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731975306448 2024-11-19T00:15:09,941 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 97fd6853d807438b9717db920ef8aadc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731975305239 2024-11-19T00:15:09,942 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea953303e13f4ef19f59243975f91dd5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731975308677 2024-11-19T00:15:09,942 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 190d3591e61447dd838c2ff6cefc2912, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731975306448 2024-11-19T00:15:09,943 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f47f1626a624f709c6993d44ef27f11, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731975308677 2024-11-19T00:15:09,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b8ea459abe0b46f28f409b09c2bb213c_999ebf86880fba150650b0d98c4af3c3 is 50, key is test_row_0/A:col10/1731975309053/Put/seqid=0 2024-11-19T00:15:09,967 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:09,977 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 999ebf86880fba150650b0d98c4af3c3#B#compaction#335 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:09,977 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/883f3d1966474795bbc0f8520db6e8af is 50, key is test_row_0/B:col10/1731975309019/Put/seqid=0 2024-11-19T00:15:09,990 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119750aaf2839194a06b418e8a3299b6602_999ebf86880fba150650b0d98c4af3c3 store=[table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:09,993 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119750aaf2839194a06b418e8a3299b6602_999ebf86880fba150650b0d98c4af3c3, store=[table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:09,993 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119750aaf2839194a06b418e8a3299b6602_999ebf86880fba150650b0d98c4af3c3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:10,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742222_1398 (size=12404) 2024-11-19T00:15:10,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:10,007 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b8ea459abe0b46f28f409b09c2bb213c_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b8ea459abe0b46f28f409b09c2bb213c_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:10,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/3809de211b694accbcc3ed2438725eb3, store: [table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:10,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/3809de211b694accbcc3ed2438725eb3 is 175, key is test_row_0/A:col10/1731975309053/Put/seqid=0 2024-11-19T00:15:10,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742223_1399 (size=12663) 2024-11-19T00:15:10,060 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/883f3d1966474795bbc0f8520db6e8af as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/883f3d1966474795bbc0f8520db6e8af 2024-11-19T00:15:10,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742224_1400 (size=4469) 2024-11-19T00:15:10,066 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 999ebf86880fba150650b0d98c4af3c3#A#compaction#334 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:10,066 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/afd29e5e57fc48508d29c9f29f68d061 is 175, key is test_row_0/A:col10/1731975309019/Put/seqid=0 2024-11-19T00:15:10,068 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 999ebf86880fba150650b0d98c4af3c3/B of 999ebf86880fba150650b0d98c4af3c3 into 883f3d1966474795bbc0f8520db6e8af(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:10,068 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:10,068 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., storeName=999ebf86880fba150650b0d98c4af3c3/B, priority=12, startTime=1731975309933; duration=0sec 2024-11-19T00:15:10,068 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:10,068 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 999ebf86880fba150650b0d98c4af3c3:B 2024-11-19T00:15:10,068 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:15:10,069 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:15:10,069 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 999ebf86880fba150650b0d98c4af3c3/C is initiating minor compaction (all files) 2024-11-19T00:15:10,069 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 999ebf86880fba150650b0d98c4af3c3/C in TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:10,069 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/4045ff10b4cc4757bcdb9696df63a33c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/37834bce25b242229f2d65230e21bf40, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/eb5f6d6f91eb4705a4810d78b3b110e8, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/4188012f0d15407dac3d2133250b4186] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp, totalSize=47.8 K 2024-11-19T00:15:10,070 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 4045ff10b4cc4757bcdb9696df63a33c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731975304868 2024-11-19T00:15:10,070 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 37834bce25b242229f2d65230e21bf40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731975305239 2024-11-19T00:15:10,070 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting eb5f6d6f91eb4705a4810d78b3b110e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731975306448 2024-11-19T00:15:10,071 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 4188012f0d15407dac3d2133250b4186, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731975308677 2024-11-19T00:15:10,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742225_1401 (size=31205) 2024-11-19T00:15:10,084 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=261, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/3809de211b694accbcc3ed2438725eb3 2024-11-19T00:15:10,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T00:15:10,117 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 999ebf86880fba150650b0d98c4af3c3#C#compaction#336 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:10,117 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/ff91275a49ab4fd6a3bbe3946837a27b is 50, key is test_row_0/C:col10/1731975309019/Put/seqid=0 2024-11-19T00:15:10,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/87bb979b43214e988c39b48d12aa33ed is 50, key is test_row_0/B:col10/1731975309053/Put/seqid=0 2024-11-19T00:15:10,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742226_1402 (size=31617) 2024-11-19T00:15:10,149 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/afd29e5e57fc48508d29c9f29f68d061 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/afd29e5e57fc48508d29c9f29f68d061 2024-11-19T00:15:10,161 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 999ebf86880fba150650b0d98c4af3c3/A of 999ebf86880fba150650b0d98c4af3c3 into afd29e5e57fc48508d29c9f29f68d061(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:10,161 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:10,161 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., storeName=999ebf86880fba150650b0d98c4af3c3/A, priority=12, startTime=1731975309932; duration=0sec 2024-11-19T00:15:10,161 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:10,161 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 999ebf86880fba150650b0d98c4af3c3:A 2024-11-19T00:15:10,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742227_1403 (size=12663) 2024-11-19T00:15:10,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742228_1404 (size=12251) 2024-11-19T00:15:10,178 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/87bb979b43214e988c39b48d12aa33ed 2024-11-19T00:15:10,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/39e7abd82ecc4e1e87f1d5060f2675fc is 50, key is test_row_0/C:col10/1731975309053/Put/seqid=0 2024-11-19T00:15:10,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742229_1405 (size=12251) 2024-11-19T00:15:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:10,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:10,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975370266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:10,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975370274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:10,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975370279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:10,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975370279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:10,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975370280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:10,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:10,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:10,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975370384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:10,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975370384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:10,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:10,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975370394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:10,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:10,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:10,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975370394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:10,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:10,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975370403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:10,572 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/ff91275a49ab4fd6a3bbe3946837a27b as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/ff91275a49ab4fd6a3bbe3946837a27b 2024-11-19T00:15:10,579 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 999ebf86880fba150650b0d98c4af3c3/C of 999ebf86880fba150650b0d98c4af3c3 into ff91275a49ab4fd6a3bbe3946837a27b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:10,579 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:10,579 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., storeName=999ebf86880fba150650b0d98c4af3c3/C, priority=12, startTime=1731975309934; duration=0sec 2024-11-19T00:15:10,579 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:10,579 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 999ebf86880fba150650b0d98c4af3c3:C 2024-11-19T00:15:10,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:10,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975370594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:10,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:10,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:10,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975370604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T00:15:10,617 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/39e7abd82ecc4e1e87f1d5060f2675fc 2024-11-19T00:15:10,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/3809de211b694accbcc3ed2438725eb3 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/3809de211b694accbcc3ed2438725eb3 2024-11-19T00:15:10,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:10,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975370615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:10,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975370616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:10,624 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/3809de211b694accbcc3ed2438725eb3, entries=150, sequenceid=261, filesize=30.5 K 2024-11-19T00:15:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:10,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/87bb979b43214e988c39b48d12aa33ed as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/87bb979b43214e988c39b48d12aa33ed 2024-11-19T00:15:10,629 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/87bb979b43214e988c39b48d12aa33ed, entries=150, sequenceid=261, filesize=12.0 K 2024-11-19T00:15:10,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/39e7abd82ecc4e1e87f1d5060f2675fc as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/39e7abd82ecc4e1e87f1d5060f2675fc 2024-11-19T00:15:10,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975370624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:10,633 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/39e7abd82ecc4e1e87f1d5060f2675fc, entries=150, sequenceid=261, filesize=12.0 K 2024-11-19T00:15:10,634 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 999ebf86880fba150650b0d98c4af3c3 in 694ms, sequenceid=261, compaction requested=false 2024-11-19T00:15:10,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:10,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:10,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-19T00:15:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-19T00:15:10,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-19T00:15:10,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1580 sec 2024-11-19T00:15:10,638 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.1630 sec 2024-11-19T00:15:10,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975310272, max timestamp=1731975310910Num uniques 859; 2024-11-19T00:15:10,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:15:10,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:10,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 999ebf86880fba150650b0d98c4af3c3 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-19T00:15:10,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=A 2024-11-19T00:15:10,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:10,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=B 2024-11-19T00:15:10,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:10,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=C 2024-11-19T00:15:10,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:10,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111984c9ce96306e491bbb3c3ee2a55b8b97_999ebf86880fba150650b0d98c4af3c3 is 50, key is test_row_0/A:col10/1731975310279/Put/seqid=0 2024-11-19T00:15:10,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742230_1406 (size=14994) 2024-11-19T00:15:10,947 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:10,960 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111984c9ce96306e491bbb3c3ee2a55b8b97_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111984c9ce96306e491bbb3c3ee2a55b8b97_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:10,961 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/5d441a0bcb7f415096973e43a8fd1206, store: [table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:10,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/5d441a0bcb7f415096973e43a8fd1206 is 175, key is test_row_0/A:col10/1731975310279/Put/seqid=0 2024-11-19T00:15:10,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742231_1407 (size=39949) 2024-11-19T00:15:10,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975370961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:10,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:10,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975370965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:10,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:10,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:10,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975370967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:10,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:10,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975370974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:10,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:10,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:10,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975370980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:10,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:10,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:11,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975371084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:11,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975371094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975371094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:11,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975371094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:11,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975371103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:11,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975371303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:11,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:11,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975371315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:11,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975371315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:11,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:11,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:11,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975371315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:11,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975371315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:11,380 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/5d441a0bcb7f415096973e43a8fd1206 2024-11-19T00:15:11,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/ac5f6dfe64cf41f5bda01073394ed103 is 50, key is test_row_0/B:col10/1731975310279/Put/seqid=0 2024-11-19T00:15:11,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742232_1408 (size=12301) 2024-11-19T00:15:11,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/ac5f6dfe64cf41f5bda01073394ed103 2024-11-19T00:15:11,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/3e33b109429d405d8fca604e650206eb is 50, key is test_row_0/C:col10/1731975310279/Put/seqid=0 2024-11-19T00:15:11,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742233_1409 (size=12301) 2024-11-19T00:15:11,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/3e33b109429d405d8fca604e650206eb 2024-11-19T00:15:11,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/5d441a0bcb7f415096973e43a8fd1206 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/5d441a0bcb7f415096973e43a8fd1206 2024-11-19T00:15:11,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/5d441a0bcb7f415096973e43a8fd1206, entries=200, sequenceid=285, filesize=39.0 K 2024-11-19T00:15:11,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/ac5f6dfe64cf41f5bda01073394ed103 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/ac5f6dfe64cf41f5bda01073394ed103 2024-11-19T00:15:11,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/ac5f6dfe64cf41f5bda01073394ed103, entries=150, sequenceid=285, filesize=12.0 K 2024-11-19T00:15:11,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/3e33b109429d405d8fca604e650206eb as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/3e33b109429d405d8fca604e650206eb 2024-11-19T00:15:11,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/3e33b109429d405d8fca604e650206eb, entries=150, sequenceid=285, filesize=12.0 K 2024-11-19T00:15:11,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for 999ebf86880fba150650b0d98c4af3c3 in 569ms, sequenceid=285, compaction requested=true 2024-11-19T00:15:11,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:11,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 999ebf86880fba150650b0d98c4af3c3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:15:11,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:11,484 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:11,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 999ebf86880fba150650b0d98c4af3c3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:15:11,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:11,484 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:11,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 999ebf86880fba150650b0d98c4af3c3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:15:11,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:11,485 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37215 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:11,485 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102771 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:11,485 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 999ebf86880fba150650b0d98c4af3c3/A is initiating minor compaction (all files) 2024-11-19T00:15:11,485 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 999ebf86880fba150650b0d98c4af3c3/B is initiating minor compaction (all files) 2024-11-19T00:15:11,485 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 999ebf86880fba150650b0d98c4af3c3/B in TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:11,485 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 999ebf86880fba150650b0d98c4af3c3/A in TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:11,485 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/883f3d1966474795bbc0f8520db6e8af, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/87bb979b43214e988c39b48d12aa33ed, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/ac5f6dfe64cf41f5bda01073394ed103] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp, totalSize=36.3 K 2024-11-19T00:15:11,485 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/afd29e5e57fc48508d29c9f29f68d061, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/3809de211b694accbcc3ed2438725eb3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/5d441a0bcb7f415096973e43a8fd1206] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp, totalSize=100.4 K 2024-11-19T00:15:11,485 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:11,485 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. files: [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/afd29e5e57fc48508d29c9f29f68d061, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/3809de211b694accbcc3ed2438725eb3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/5d441a0bcb7f415096973e43a8fd1206] 2024-11-19T00:15:11,485 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 883f3d1966474795bbc0f8520db6e8af, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731975308677 2024-11-19T00:15:11,485 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting afd29e5e57fc48508d29c9f29f68d061, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731975308677 2024-11-19T00:15:11,485 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 87bb979b43214e988c39b48d12aa33ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731975309042 2024-11-19T00:15:11,485 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3809de211b694accbcc3ed2438725eb3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731975309042 2024-11-19T00:15:11,486 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting ac5f6dfe64cf41f5bda01073394ed103, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731975310277 2024-11-19T00:15:11,486 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d441a0bcb7f415096973e43a8fd1206, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731975310272 2024-11-19T00:15:11,503 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 999ebf86880fba150650b0d98c4af3c3#B#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:11,503 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:11,504 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/aa2483d3c98f405a82eca3c6b182e8f3 is 50, key is test_row_0/B:col10/1731975310279/Put/seqid=0 2024-11-19T00:15:11,506 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119d400b2c7bec64381a5643d742adac588_999ebf86880fba150650b0d98c4af3c3 store=[table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:11,508 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119d400b2c7bec64381a5643d742adac588_999ebf86880fba150650b0d98c4af3c3, store=[table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:11,508 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d400b2c7bec64381a5643d742adac588_999ebf86880fba150650b0d98c4af3c3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:11,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742234_1410 (size=12915) 2024-11-19T00:15:11,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742235_1411 (size=4469) 2024-11-19T00:15:11,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T00:15:11,634 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:15:11,634 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:15:11,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-19T00:15:11,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T00:15:11,636 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:15:11,637 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:15:11,637 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:15:11,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:11,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 999ebf86880fba150650b0d98c4af3c3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T00:15:11,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=A 2024-11-19T00:15:11,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:11,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=B 2024-11-19T00:15:11,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:11,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=C 2024-11-19T00:15:11,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:11,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119211bd125ce6c4b29a17e9aba31ff660e_999ebf86880fba150650b0d98c4af3c3 is 50, key is test_row_0/A:col10/1731975311640/Put/seqid=0 2024-11-19T00:15:11,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742236_1412 (size=14994) 2024-11-19T00:15:11,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:11,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975371657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:11,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:11,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975371665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:11,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975371665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:11,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975371670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:11,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:11,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975371674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:11,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:11,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T00:15:11,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T00:15:11,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:11,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:11,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:11,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:11,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:11,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975371784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:11,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975371784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:11,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975371793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:11,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975371793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:11,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:11,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975371793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:11,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:11,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:11,920 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/aa2483d3c98f405a82eca3c6b182e8f3 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/aa2483d3c98f405a82eca3c6b182e8f3 2024-11-19T00:15:11,923 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 999ebf86880fba150650b0d98c4af3c3#A#compaction#343 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:11,923 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/fc7c8276df5a4e38b103133bff4bedf5 is 175, key is test_row_0/A:col10/1731975310279/Put/seqid=0 2024-11-19T00:15:11,924 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 999ebf86880fba150650b0d98c4af3c3/B of 999ebf86880fba150650b0d98c4af3c3 into aa2483d3c98f405a82eca3c6b182e8f3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:11,925 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:11,925 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., storeName=999ebf86880fba150650b0d98c4af3c3/B, priority=13, startTime=1731975311484; duration=0sec 2024-11-19T00:15:11,925 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:11,925 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 999ebf86880fba150650b0d98c4af3c3:B 2024-11-19T00:15:11,925 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:11,926 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37215 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:11,926 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 999ebf86880fba150650b0d98c4af3c3/C is initiating minor compaction (all files) 2024-11-19T00:15:11,926 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 999ebf86880fba150650b0d98c4af3c3/C in TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:11,926 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/ff91275a49ab4fd6a3bbe3946837a27b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/39e7abd82ecc4e1e87f1d5060f2675fc, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/3e33b109429d405d8fca604e650206eb] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp, totalSize=36.3 K 2024-11-19T00:15:11,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742237_1413 (size=31869) 2024-11-19T00:15:11,927 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting ff91275a49ab4fd6a3bbe3946837a27b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731975308677 2024-11-19T00:15:11,927 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 39e7abd82ecc4e1e87f1d5060f2675fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731975309042 2024-11-19T00:15:11,928 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e33b109429d405d8fca604e650206eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731975310277 2024-11-19T00:15:11,931 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/fc7c8276df5a4e38b103133bff4bedf5 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/fc7c8276df5a4e38b103133bff4bedf5 2024-11-19T00:15:11,934 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 999ebf86880fba150650b0d98c4af3c3#C#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:11,935 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/60e122daf3f34a15bfcb380c1d214b8d is 50, key is test_row_0/C:col10/1731975310279/Put/seqid=0 2024-11-19T00:15:11,936 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 999ebf86880fba150650b0d98c4af3c3/A of 999ebf86880fba150650b0d98c4af3c3 into fc7c8276df5a4e38b103133bff4bedf5(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:11,936 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:11,936 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., storeName=999ebf86880fba150650b0d98c4af3c3/A, priority=13, startTime=1731975311484; duration=0sec 2024-11-19T00:15:11,936 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:11,936 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 999ebf86880fba150650b0d98c4af3c3:A 2024-11-19T00:15:11,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742238_1414 (size=12915) 2024-11-19T00:15:11,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T00:15:11,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:11,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:11,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:11,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:11,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:11,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T00:15:11,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975371994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:11,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:11,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:12,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975372003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:12,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:12,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975372013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:12,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:12,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975372014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:12,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:12,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975372014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:12,065 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:12,068 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119211bd125ce6c4b29a17e9aba31ff660e_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119211bd125ce6c4b29a17e9aba31ff660e_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:12,068 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/99d9b75289ab4c0e9f06a44efbc9a29f, store: [table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:12,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/99d9b75289ab4c0e9f06a44efbc9a29f is 175, key is test_row_0/A:col10/1731975311640/Put/seqid=0 2024-11-19T00:15:12,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742239_1415 (size=39949) 2024-11-19T00:15:12,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T00:15:12,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:12,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:12,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:12,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:12,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:12,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T00:15:12,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:12,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:12,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:12,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:12,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:12,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T00:15:12,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975372303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:12,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975372323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:12,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:12,342 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/60e122daf3f34a15bfcb380c1d214b8d as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/60e122daf3f34a15bfcb380c1d214b8d 2024-11-19T00:15:12,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:12,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975372334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975372334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975372334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:12,347 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 999ebf86880fba150650b0d98c4af3c3/C of 999ebf86880fba150650b0d98c4af3c3 into 60e122daf3f34a15bfcb380c1d214b8d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:12,347 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:12,347 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., storeName=999ebf86880fba150650b0d98c4af3c3/C, priority=13, startTime=1731975311484; duration=0sec 2024-11-19T00:15:12,347 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:12,347 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 999ebf86880fba150650b0d98c4af3c3:C 2024-11-19T00:15:12,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T00:15:12,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:12,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:12,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:12,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:12,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:12,474 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=302, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/99d9b75289ab4c0e9f06a44efbc9a29f 2024-11-19T00:15:12,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/01467ef6836b4c538bb0935939c4448c is 50, key is test_row_0/B:col10/1731975311640/Put/seqid=0 2024-11-19T00:15:12,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742240_1416 (size=12301) 2024-11-19T00:15:12,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T00:15:12,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:12,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:12,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:12,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:12,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:12,652 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T00:15:12,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T00:15:12,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:12,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:12,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:12,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:12,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:12,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:12,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T00:15:12,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:12,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975372824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:12,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:12,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:12,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975372843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:12,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:12,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T00:15:12,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:12,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:12,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:12,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:12,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:12,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975372853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:12,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:12,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975372853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:12,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:12,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975372854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:12,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:12,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/01467ef6836b4c538bb0935939c4448c 2024-11-19T00:15:12,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/fa1ef28d1b8342ec949877be056c5802 is 50, key is test_row_0/C:col10/1731975311640/Put/seqid=0 2024-11-19T00:15:12,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742241_1417 (size=12301) 2024-11-19T00:15:13,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T00:15:13,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:13,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:13,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:13,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:13,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:13,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:13,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T00:15:13,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:13,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:13,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:13,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:13,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:13,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:13,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/fa1ef28d1b8342ec949877be056c5802 2024-11-19T00:15:13,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/99d9b75289ab4c0e9f06a44efbc9a29f as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/99d9b75289ab4c0e9f06a44efbc9a29f 2024-11-19T00:15:13,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/99d9b75289ab4c0e9f06a44efbc9a29f, entries=200, sequenceid=302, filesize=39.0 K 2024-11-19T00:15:13,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/01467ef6836b4c538bb0935939c4448c as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/01467ef6836b4c538bb0935939c4448c 2024-11-19T00:15:13,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/01467ef6836b4c538bb0935939c4448c, entries=150, sequenceid=302, filesize=12.0 K 2024-11-19T00:15:13,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/fa1ef28d1b8342ec949877be056c5802 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/fa1ef28d1b8342ec949877be056c5802 2024-11-19T00:15:13,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/fa1ef28d1b8342ec949877be056c5802, entries=150, sequenceid=302, filesize=12.0 K 2024-11-19T00:15:13,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 999ebf86880fba150650b0d98c4af3c3 in 1670ms, sequenceid=302, compaction requested=false 2024-11-19T00:15:13,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:13,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T00:15:13,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:13,318 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 999ebf86880fba150650b0d98c4af3c3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T00:15:13,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=A 2024-11-19T00:15:13,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:13,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=B 2024-11-19T00:15:13,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:13,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=C 2024-11-19T00:15:13,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:13,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119842ad28d37504f618df56a4c207dd9b1_999ebf86880fba150650b0d98c4af3c3 is 50, key is test_row_0/A:col10/1731975311654/Put/seqid=0 2024-11-19T00:15:13,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742242_1418 (size=12454) 2024-11-19T00:15:13,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:13,735 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119842ad28d37504f618df56a4c207dd9b1_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119842ad28d37504f618df56a4c207dd9b1_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:13,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/97a76943ef564e689ef62796c305f308, store: [table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:13,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/97a76943ef564e689ef62796c305f308 is 175, key is test_row_0/A:col10/1731975311654/Put/seqid=0 2024-11-19T00:15:13,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742243_1419 (size=31255) 2024-11-19T00:15:13,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T00:15:13,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:13,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. as already flushing 2024-11-19T00:15:13,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:13,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975373874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:13,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:13,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975373876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:13,885 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_1' in 999ebf86880fba150650b0d98c4af3c3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:15:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:13,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:13,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975373881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:13,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:13,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:13,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:13,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:13,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975373882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:13,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:13,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:13,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:13,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975373885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:13,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:13,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:13,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:13,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:13,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:13,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975373994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:13,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:13,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975373994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:13,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:13,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:13,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:13,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:13,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:13,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:14,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:14,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975374003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:14,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975374003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:14,147 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/97a76943ef564e689ef62796c305f308 2024-11-19T00:15:14,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/8e04d8de7f3347fcb18fb7a265bc7cbe is 50, key is test_row_0/B:col10/1731975311654/Put/seqid=0 2024-11-19T00:15:14,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742244_1420 (size=12301) 2024-11-19T00:15:14,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975374204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:14,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975374204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:14,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:14,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:14,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:14,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:14,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:14,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:14,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:14,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:14,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975374224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:14,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975374224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:14,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975374514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:14,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:14,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975374514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:14,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:14,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975374543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:14,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:14,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:14,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:14,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:14,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975374543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:14,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:14,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:14,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:14,557 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/8e04d8de7f3347fcb18fb7a265bc7cbe 2024-11-19T00:15:14,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/e02ee7c18a3346859feeb288306a659e is 50, key is test_row_0/C:col10/1731975311654/Put/seqid=0 2024-11-19T00:15:14,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742245_1421 (size=12301) 2024-11-19T00:15:14,570 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/e02ee7c18a3346859feeb288306a659e 2024-11-19T00:15:14,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/97a76943ef564e689ef62796c305f308 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/97a76943ef564e689ef62796c305f308 2024-11-19T00:15:14,577 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/97a76943ef564e689ef62796c305f308, entries=150, sequenceid=320, filesize=30.5 K 2024-11-19T00:15:14,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/8e04d8de7f3347fcb18fb7a265bc7cbe as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/8e04d8de7f3347fcb18fb7a265bc7cbe 2024-11-19T00:15:14,580 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/8e04d8de7f3347fcb18fb7a265bc7cbe, entries=150, sequenceid=320, filesize=12.0 K 2024-11-19T00:15:14,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/e02ee7c18a3346859feeb288306a659e as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/e02ee7c18a3346859feeb288306a659e 2024-11-19T00:15:14,584 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/e02ee7c18a3346859feeb288306a659e, entries=150, sequenceid=320, filesize=12.0 K 2024-11-19T00:15:14,584 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 999ebf86880fba150650b0d98c4af3c3 in 1266ms, sequenceid=320, compaction requested=true 2024-11-19T00:15:14,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:14,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:14,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-19T00:15:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-19T00:15:14,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-19T00:15:14,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9490 sec 2024-11-19T00:15:14,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.9530 sec 2024-11-19T00:15:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:15,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 999ebf86880fba150650b0d98c4af3c3 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T00:15:15,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=A 2024-11-19T00:15:15,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:15,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=B 2024-11-19T00:15:15,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:15,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=C 2024-11-19T00:15:15,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:15,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c78872cc6ab64475a81166f274896ebb_999ebf86880fba150650b0d98c4af3c3 is 50, key is test_row_0/A:col10/1731975315030/Put/seqid=0 2024-11-19T00:15:15,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742246_1422 (size=14994) 2024-11-19T00:15:15,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:15,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975375063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:15,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:15,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:15,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975375064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:15,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975375065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:15,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975375067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:15,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:15,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975375184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:15,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975375184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975375184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975375184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:15,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:15,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975375394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:15,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:15,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975375404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:15,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975375404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:15,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975375404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:15,445 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:15,448 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c78872cc6ab64475a81166f274896ebb_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c78872cc6ab64475a81166f274896ebb_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:15,448 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/14f15acc4c9240feb0300c7b5cb79983, store: [table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:15,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/14f15acc4c9240feb0300c7b5cb79983 is 175, key is test_row_0/A:col10/1731975315030/Put/seqid=0 2024-11-19T00:15:15,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742247_1423 (size=39949) 2024-11-19T00:15:15,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975375704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:15,715 INFO [Thread-1551 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1551. 2024-11-19T00:15:15,716 DEBUG [Thread-1551 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:15:15,716 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:15,716 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:15,716 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:15:15,723 INFO [Thread-1559 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1559. 2024-11-19T00:15:15,723 DEBUG [Thread-1559 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:15:15,723 DEBUG [Thread-1559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:15,723 DEBUG [Thread-1559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:15,724 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:15:15,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:15,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975375724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:15,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975375725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975375726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:15,726 INFO [Thread-1557 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1557. 2024-11-19T00:15:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,726 DEBUG [Thread-1557 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:15:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,726 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:15,727 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:15,727 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:15:15,731 INFO [Thread-1555 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1555. 2024-11-19T00:15:15,731 DEBUG [Thread-1555 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:15:15,731 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:15,731 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:15,732 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:15:15,737 INFO [Thread-1553 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1553. 2024-11-19T00:15:15,738 DEBUG [Thread-1553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:15:15,738 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:15,738 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:15,738 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:15:15,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T00:15:15,804 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:15:15,853 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=338, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/14f15acc4c9240feb0300c7b5cb79983 2024-11-19T00:15:15,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/41efcba2eb484f96a2ee2c32600c1682 is 50, key is test_row_0/B:col10/1731975315030/Put/seqid=0 2024-11-19T00:15:15,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742248_1424 (size=12301) 2024-11-19T00:15:15,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:15,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50892 deadline: 1731975375893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:15,895 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_1' in 999ebf86880fba150650b0d98c4af3c3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:15:15,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:15,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:16,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:16,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50808 deadline: 1731975376223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:16,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:16,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:16,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:16,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:16,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:16,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50888 deadline: 1731975376234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50882 deadline: 1731975376234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50850 deadline: 1731975376234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:16,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:16,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:16,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:16,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:16,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:16,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:16,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:16,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:16,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:16,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/41efcba2eb484f96a2ee2c32600c1682 2024-11-19T00:15:16,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/b28b2d5be6244d238fa47417375edf39 is 50, key is test_row_0/C:col10/1731975315030/Put/seqid=0 2024-11-19T00:15:16,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742249_1425 (size=12301) 2024-11-19T00:15:16,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/b28b2d5be6244d238fa47417375edf39 2024-11-19T00:15:16,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/14f15acc4c9240feb0300c7b5cb79983 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/14f15acc4c9240feb0300c7b5cb79983 2024-11-19T00:15:16,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/14f15acc4c9240feb0300c7b5cb79983, entries=200, sequenceid=338, filesize=39.0 K 2024-11-19T00:15:16,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/41efcba2eb484f96a2ee2c32600c1682 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/41efcba2eb484f96a2ee2c32600c1682 2024-11-19T00:15:16,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/41efcba2eb484f96a2ee2c32600c1682, entries=150, sequenceid=338, filesize=12.0 K 2024-11-19T00:15:16,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/b28b2d5be6244d238fa47417375edf39 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/b28b2d5be6244d238fa47417375edf39 2024-11-19T00:15:16,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/b28b2d5be6244d238fa47417375edf39, entries=150, sequenceid=338, filesize=12.0 K 2024-11-19T00:15:16,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 999ebf86880fba150650b0d98c4af3c3 in 1679ms, sequenceid=338, compaction requested=true 2024-11-19T00:15:16,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:16,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 999ebf86880fba150650b0d98c4af3c3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:15:16,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:16,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 999ebf86880fba150650b0d98c4af3c3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:15:16,711 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:15:16,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:16,711 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:15:16,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 999ebf86880fba150650b0d98c4af3c3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:15:16,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:16,712 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:15:16,712 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:15:16,712 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 999ebf86880fba150650b0d98c4af3c3/B is initiating minor compaction (all files) 2024-11-19T00:15:16,712 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 999ebf86880fba150650b0d98c4af3c3/A is initiating minor compaction (all files) 2024-11-19T00:15:16,712 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 999ebf86880fba150650b0d98c4af3c3/B in TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:16,712 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 999ebf86880fba150650b0d98c4af3c3/A in TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:16,712 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/aa2483d3c98f405a82eca3c6b182e8f3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/01467ef6836b4c538bb0935939c4448c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/8e04d8de7f3347fcb18fb7a265bc7cbe, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/41efcba2eb484f96a2ee2c32600c1682] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp, totalSize=48.7 K 2024-11-19T00:15:16,712 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/fc7c8276df5a4e38b103133bff4bedf5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/99d9b75289ab4c0e9f06a44efbc9a29f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/97a76943ef564e689ef62796c305f308, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/14f15acc4c9240feb0300c7b5cb79983] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp, totalSize=139.7 K 2024-11-19T00:15:16,712 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:16,712 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. files: [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/fc7c8276df5a4e38b103133bff4bedf5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/99d9b75289ab4c0e9f06a44efbc9a29f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/97a76943ef564e689ef62796c305f308, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/14f15acc4c9240feb0300c7b5cb79983] 2024-11-19T00:15:16,712 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting aa2483d3c98f405a82eca3c6b182e8f3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731975310277 2024-11-19T00:15:16,713 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 01467ef6836b4c538bb0935939c4448c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731975310964 2024-11-19T00:15:16,713 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc7c8276df5a4e38b103133bff4bedf5, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731975310277 2024-11-19T00:15:16,713 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e04d8de7f3347fcb18fb7a265bc7cbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731975311654 2024-11-19T00:15:16,713 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99d9b75289ab4c0e9f06a44efbc9a29f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731975310964 2024-11-19T00:15:16,713 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 41efcba2eb484f96a2ee2c32600c1682, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731975313867 2024-11-19T00:15:16,713 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97a76943ef564e689ef62796c305f308, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731975311654 2024-11-19T00:15:16,713 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14f15acc4c9240feb0300c7b5cb79983, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731975313867 2024-11-19T00:15:16,719 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:16,727 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119ec91a4db926342dca3efe216e7d1efa6_999ebf86880fba150650b0d98c4af3c3 store=[table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:16,728 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 999ebf86880fba150650b0d98c4af3c3#B#compaction#355 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:16,728 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/c1b3967dd1264909956f3f41d692c081 is 50, key is test_row_0/B:col10/1731975315030/Put/seqid=0 2024-11-19T00:15:16,734 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119ec91a4db926342dca3efe216e7d1efa6_999ebf86880fba150650b0d98c4af3c3, store=[table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:16,734 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ec91a4db926342dca3efe216e7d1efa6_999ebf86880fba150650b0d98c4af3c3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:16,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742250_1426 (size=13051) 2024-11-19T00:15:16,768 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/c1b3967dd1264909956f3f41d692c081 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/c1b3967dd1264909956f3f41d692c081 2024-11-19T00:15:16,774 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 999ebf86880fba150650b0d98c4af3c3/B of 999ebf86880fba150650b0d98c4af3c3 into c1b3967dd1264909956f3f41d692c081(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:16,774 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:16,774 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., storeName=999ebf86880fba150650b0d98c4af3c3/B, priority=12, startTime=1731975316711; duration=0sec 2024-11-19T00:15:16,775 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:16,775 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 999ebf86880fba150650b0d98c4af3c3:B 2024-11-19T00:15:16,775 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:15:16,776 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:15:16,776 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 999ebf86880fba150650b0d98c4af3c3/C is initiating minor compaction (all files) 2024-11-19T00:15:16,776 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 999ebf86880fba150650b0d98c4af3c3/C in TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:16,777 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/60e122daf3f34a15bfcb380c1d214b8d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/fa1ef28d1b8342ec949877be056c5802, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/e02ee7c18a3346859feeb288306a659e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/b28b2d5be6244d238fa47417375edf39] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp, totalSize=48.7 K 2024-11-19T00:15:16,777 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 60e122daf3f34a15bfcb380c1d214b8d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731975310277 2024-11-19T00:15:16,777 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting fa1ef28d1b8342ec949877be056c5802, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731975310964 2024-11-19T00:15:16,778 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting e02ee7c18a3346859feeb288306a659e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731975311654 2024-11-19T00:15:16,778 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting b28b2d5be6244d238fa47417375edf39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731975313867 2024-11-19T00:15:16,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742251_1427 (size=4469) 2024-11-19T00:15:16,782 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 999ebf86880fba150650b0d98c4af3c3#A#compaction#354 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:16,783 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/7353fb369c80412b994d90878c350205 is 175, key is test_row_0/A:col10/1731975315030/Put/seqid=0 2024-11-19T00:15:16,810 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 999ebf86880fba150650b0d98c4af3c3#C#compaction#356 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:16,811 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/05306509d7494634ab33b8e158522d61 is 50, key is test_row_0/C:col10/1731975315030/Put/seqid=0 2024-11-19T00:15:16,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742252_1428 (size=32005) 2024-11-19T00:15:16,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742253_1429 (size=13051) 2024-11-19T00:15:16,826 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/7353fb369c80412b994d90878c350205 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/7353fb369c80412b994d90878c350205 2024-11-19T00:15:16,827 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/05306509d7494634ab33b8e158522d61 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/05306509d7494634ab33b8e158522d61 2024-11-19T00:15:16,833 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 999ebf86880fba150650b0d98c4af3c3/A of 999ebf86880fba150650b0d98c4af3c3 into 7353fb369c80412b994d90878c350205(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:16,833 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:16,833 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., storeName=999ebf86880fba150650b0d98c4af3c3/A, priority=12, startTime=1731975316711; duration=0sec 2024-11-19T00:15:16,833 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:16,833 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 999ebf86880fba150650b0d98c4af3c3:A 2024-11-19T00:15:16,836 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 999ebf86880fba150650b0d98c4af3c3/C of 999ebf86880fba150650b0d98c4af3c3 into 05306509d7494634ab33b8e158522d61(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:16,836 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:16,836 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., storeName=999ebf86880fba150650b0d98c4af3c3/C, priority=12, startTime=1731975316711; duration=0sec 2024-11-19T00:15:16,837 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:16,837 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 999ebf86880fba150650b0d98c4af3c3:C 2024-11-19T00:15:17,023 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/ef00bb5e8f284f3a8c8e2db827b39993, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/1d562e9d38964d0fa7b4e547d7aaa824, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/1b401825aad64b7a8f078109a2d5d6b5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/27acba2b420f47adbd471256b4389690, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/77bad13e75e245cd9da2aeec36429406, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/7e8d97a4ea644ba79c1848ce9431521e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/3362e338836c4d2588f4d7a06a021345, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/b6fe8cd07c454839a7135199b7a8aff9, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/3aa8ea01b5d34f64b9b24af45f84804f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/96bb017918f5458e97913432f4270a08, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/c59aae861f0d4fe58bc7ad5171522a3e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/d818d93551e74f17a2935e785de8f95a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/114dfc8ae90e4dc4ac7006967a20aa3a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/ed20d48bd0264adbb33d70119f51769f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/ea953303e13f4ef19f59243975f91dd5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/afd29e5e57fc48508d29c9f29f68d061, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/3809de211b694accbcc3ed2438725eb3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/5d441a0bcb7f415096973e43a8fd1206, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/fc7c8276df5a4e38b103133bff4bedf5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/99d9b75289ab4c0e9f06a44efbc9a29f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/97a76943ef564e689ef62796c305f308, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/14f15acc4c9240feb0300c7b5cb79983] to archive 2024-11-19T00:15:17,024 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T00:15:17,025 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/ef00bb5e8f284f3a8c8e2db827b39993 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/ef00bb5e8f284f3a8c8e2db827b39993 2024-11-19T00:15:17,026 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/1d562e9d38964d0fa7b4e547d7aaa824 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/1d562e9d38964d0fa7b4e547d7aaa824 2024-11-19T00:15:17,027 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/1b401825aad64b7a8f078109a2d5d6b5 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/1b401825aad64b7a8f078109a2d5d6b5 2024-11-19T00:15:17,028 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/27acba2b420f47adbd471256b4389690 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/27acba2b420f47adbd471256b4389690 2024-11-19T00:15:17,029 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/77bad13e75e245cd9da2aeec36429406 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/77bad13e75e245cd9da2aeec36429406 2024-11-19T00:15:17,029 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/7e8d97a4ea644ba79c1848ce9431521e to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/7e8d97a4ea644ba79c1848ce9431521e 2024-11-19T00:15:17,030 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/3362e338836c4d2588f4d7a06a021345 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/3362e338836c4d2588f4d7a06a021345 2024-11-19T00:15:17,031 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/b6fe8cd07c454839a7135199b7a8aff9 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/b6fe8cd07c454839a7135199b7a8aff9 2024-11-19T00:15:17,033 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/3aa8ea01b5d34f64b9b24af45f84804f to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/3aa8ea01b5d34f64b9b24af45f84804f 2024-11-19T00:15:17,034 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/96bb017918f5458e97913432f4270a08 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/96bb017918f5458e97913432f4270a08 2024-11-19T00:15:17,035 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/c59aae861f0d4fe58bc7ad5171522a3e to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/c59aae861f0d4fe58bc7ad5171522a3e 2024-11-19T00:15:17,035 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/d818d93551e74f17a2935e785de8f95a to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/d818d93551e74f17a2935e785de8f95a 2024-11-19T00:15:17,036 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/114dfc8ae90e4dc4ac7006967a20aa3a to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/114dfc8ae90e4dc4ac7006967a20aa3a 2024-11-19T00:15:17,037 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/ed20d48bd0264adbb33d70119f51769f to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/ed20d48bd0264adbb33d70119f51769f 2024-11-19T00:15:17,038 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/ea953303e13f4ef19f59243975f91dd5 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/ea953303e13f4ef19f59243975f91dd5 2024-11-19T00:15:17,039 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/afd29e5e57fc48508d29c9f29f68d061 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/afd29e5e57fc48508d29c9f29f68d061 2024-11-19T00:15:17,040 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/3809de211b694accbcc3ed2438725eb3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/3809de211b694accbcc3ed2438725eb3 2024-11-19T00:15:17,041 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/5d441a0bcb7f415096973e43a8fd1206 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/5d441a0bcb7f415096973e43a8fd1206 2024-11-19T00:15:17,043 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/fc7c8276df5a4e38b103133bff4bedf5 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/fc7c8276df5a4e38b103133bff4bedf5 2024-11-19T00:15:17,044 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/99d9b75289ab4c0e9f06a44efbc9a29f to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/99d9b75289ab4c0e9f06a44efbc9a29f 2024-11-19T00:15:17,045 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/97a76943ef564e689ef62796c305f308 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/97a76943ef564e689ef62796c305f308 2024-11-19T00:15:17,046 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/14f15acc4c9240feb0300c7b5cb79983 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/14f15acc4c9240feb0300c7b5cb79983 2024-11-19T00:15:17,050 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/429f2da816ab4ccd94da6e0cec887dd6, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/c9b19a9cf190495eb409f9312aab9358, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/d5d0d142e9454670b35fa46f6ae38422, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/1585c43935ae4b75ba3e05e01e614e94, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/e0b3ef8d81e14f1cbcda495e6cd81a98, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/d21593f8e9c24b45b1dc900ba1a4b255, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/d0bb80a5b0f94030b3bcbebf0cefc616, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/9685057c5fd2466daf7895b8c36ea124, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/29c68ac57bc0460482b1bd434c41943b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/0e96da03a4a94c0da0a264e7bfbee2a2, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/d8d493eacfc34810ac87a514949e8edb, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/52577767d86747d1acb62a9c8784559c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/97fd6853d807438b9717db920ef8aadc, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/190d3591e61447dd838c2ff6cefc2912, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/883f3d1966474795bbc0f8520db6e8af, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/1f47f1626a624f709c6993d44ef27f11, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/87bb979b43214e988c39b48d12aa33ed, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/aa2483d3c98f405a82eca3c6b182e8f3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/ac5f6dfe64cf41f5bda01073394ed103, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/01467ef6836b4c538bb0935939c4448c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/8e04d8de7f3347fcb18fb7a265bc7cbe, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/41efcba2eb484f96a2ee2c32600c1682] to archive 2024-11-19T00:15:17,051 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T00:15:17,053 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/429f2da816ab4ccd94da6e0cec887dd6 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/429f2da816ab4ccd94da6e0cec887dd6 2024-11-19T00:15:17,054 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/c9b19a9cf190495eb409f9312aab9358 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/c9b19a9cf190495eb409f9312aab9358 2024-11-19T00:15:17,055 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/d5d0d142e9454670b35fa46f6ae38422 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/d5d0d142e9454670b35fa46f6ae38422 2024-11-19T00:15:17,056 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/1585c43935ae4b75ba3e05e01e614e94 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/1585c43935ae4b75ba3e05e01e614e94 2024-11-19T00:15:17,057 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/e0b3ef8d81e14f1cbcda495e6cd81a98 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/e0b3ef8d81e14f1cbcda495e6cd81a98 2024-11-19T00:15:17,062 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/d21593f8e9c24b45b1dc900ba1a4b255 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/d21593f8e9c24b45b1dc900ba1a4b255 2024-11-19T00:15:17,063 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/d0bb80a5b0f94030b3bcbebf0cefc616 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/d0bb80a5b0f94030b3bcbebf0cefc616 2024-11-19T00:15:17,064 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/9685057c5fd2466daf7895b8c36ea124 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/9685057c5fd2466daf7895b8c36ea124 2024-11-19T00:15:17,065 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/29c68ac57bc0460482b1bd434c41943b to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/29c68ac57bc0460482b1bd434c41943b 2024-11-19T00:15:17,067 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/0e96da03a4a94c0da0a264e7bfbee2a2 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/0e96da03a4a94c0da0a264e7bfbee2a2 2024-11-19T00:15:17,070 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/d8d493eacfc34810ac87a514949e8edb to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/d8d493eacfc34810ac87a514949e8edb 2024-11-19T00:15:17,071 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/52577767d86747d1acb62a9c8784559c to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/52577767d86747d1acb62a9c8784559c 2024-11-19T00:15:17,072 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/97fd6853d807438b9717db920ef8aadc to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/97fd6853d807438b9717db920ef8aadc 2024-11-19T00:15:17,073 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/190d3591e61447dd838c2ff6cefc2912 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/190d3591e61447dd838c2ff6cefc2912 2024-11-19T00:15:17,074 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/883f3d1966474795bbc0f8520db6e8af to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/883f3d1966474795bbc0f8520db6e8af 2024-11-19T00:15:17,075 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/1f47f1626a624f709c6993d44ef27f11 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/1f47f1626a624f709c6993d44ef27f11 2024-11-19T00:15:17,076 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/87bb979b43214e988c39b48d12aa33ed to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/87bb979b43214e988c39b48d12aa33ed 2024-11-19T00:15:17,077 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/aa2483d3c98f405a82eca3c6b182e8f3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/aa2483d3c98f405a82eca3c6b182e8f3 2024-11-19T00:15:17,078 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/ac5f6dfe64cf41f5bda01073394ed103 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/ac5f6dfe64cf41f5bda01073394ed103 2024-11-19T00:15:17,081 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/01467ef6836b4c538bb0935939c4448c to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/01467ef6836b4c538bb0935939c4448c 2024-11-19T00:15:17,082 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/8e04d8de7f3347fcb18fb7a265bc7cbe to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/8e04d8de7f3347fcb18fb7a265bc7cbe 2024-11-19T00:15:17,084 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/41efcba2eb484f96a2ee2c32600c1682 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/41efcba2eb484f96a2ee2c32600c1682 2024-11-19T00:15:17,089 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/ba260e839d2c4bbc8717ef9bd54d4fd6, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/fccf6507d2804855b4511c76ad009113, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/c25053a2def74d0eafd397931d216318, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/d451237c443c4238909bba1becdbb286, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/2174cb09b6404526995f006fe67b1252, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/d4c9f96447b64954afc715ae844c2dfe, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/f914d01b537a4b2883264841f6f1074e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/435831515a5b44279cddb5b6b2c84969, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/1719ec3be94048dfb94834bb6b0e280e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/7332efe7d2f04e61a7cd3403d3917ed7, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/4045ff10b4cc4757bcdb9696df63a33c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/424e63ba321d4a2aaca18ea951888730, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/37834bce25b242229f2d65230e21bf40, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/eb5f6d6f91eb4705a4810d78b3b110e8, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/ff91275a49ab4fd6a3bbe3946837a27b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/4188012f0d15407dac3d2133250b4186, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/39e7abd82ecc4e1e87f1d5060f2675fc, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/60e122daf3f34a15bfcb380c1d214b8d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/3e33b109429d405d8fca604e650206eb, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/fa1ef28d1b8342ec949877be056c5802, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/e02ee7c18a3346859feeb288306a659e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/b28b2d5be6244d238fa47417375edf39] to archive 2024-11-19T00:15:17,090 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T00:15:17,092 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/ba260e839d2c4bbc8717ef9bd54d4fd6 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/ba260e839d2c4bbc8717ef9bd54d4fd6 2024-11-19T00:15:17,093 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/fccf6507d2804855b4511c76ad009113 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/fccf6507d2804855b4511c76ad009113 2024-11-19T00:15:17,094 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/c25053a2def74d0eafd397931d216318 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/c25053a2def74d0eafd397931d216318 2024-11-19T00:15:17,095 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/d451237c443c4238909bba1becdbb286 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/d451237c443c4238909bba1becdbb286 2024-11-19T00:15:17,096 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/2174cb09b6404526995f006fe67b1252 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/2174cb09b6404526995f006fe67b1252 2024-11-19T00:15:17,097 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/d4c9f96447b64954afc715ae844c2dfe to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/d4c9f96447b64954afc715ae844c2dfe 2024-11-19T00:15:17,098 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/f914d01b537a4b2883264841f6f1074e to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/f914d01b537a4b2883264841f6f1074e 2024-11-19T00:15:17,099 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/435831515a5b44279cddb5b6b2c84969 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/435831515a5b44279cddb5b6b2c84969 2024-11-19T00:15:17,101 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/1719ec3be94048dfb94834bb6b0e280e to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/1719ec3be94048dfb94834bb6b0e280e 2024-11-19T00:15:17,102 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/7332efe7d2f04e61a7cd3403d3917ed7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/7332efe7d2f04e61a7cd3403d3917ed7 2024-11-19T00:15:17,103 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/4045ff10b4cc4757bcdb9696df63a33c to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/4045ff10b4cc4757bcdb9696df63a33c 2024-11-19T00:15:17,104 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/424e63ba321d4a2aaca18ea951888730 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/424e63ba321d4a2aaca18ea951888730 2024-11-19T00:15:17,105 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/37834bce25b242229f2d65230e21bf40 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/37834bce25b242229f2d65230e21bf40 2024-11-19T00:15:17,106 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/eb5f6d6f91eb4705a4810d78b3b110e8 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/eb5f6d6f91eb4705a4810d78b3b110e8 2024-11-19T00:15:17,107 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/ff91275a49ab4fd6a3bbe3946837a27b to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/ff91275a49ab4fd6a3bbe3946837a27b 2024-11-19T00:15:17,109 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/4188012f0d15407dac3d2133250b4186 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/4188012f0d15407dac3d2133250b4186 2024-11-19T00:15:17,117 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/39e7abd82ecc4e1e87f1d5060f2675fc to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/39e7abd82ecc4e1e87f1d5060f2675fc 2024-11-19T00:15:17,119 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/60e122daf3f34a15bfcb380c1d214b8d to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/60e122daf3f34a15bfcb380c1d214b8d 2024-11-19T00:15:17,121 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/3e33b109429d405d8fca604e650206eb to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/3e33b109429d405d8fca604e650206eb 2024-11-19T00:15:17,123 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/fa1ef28d1b8342ec949877be056c5802 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/fa1ef28d1b8342ec949877be056c5802 2024-11-19T00:15:17,125 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/e02ee7c18a3346859feeb288306a659e to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/e02ee7c18a3346859feeb288306a659e 2024-11-19T00:15:17,127 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6b1531871d35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/b28b2d5be6244d238fa47417375edf39 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/b28b2d5be6244d238fa47417375edf39 2024-11-19T00:15:17,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:17,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 999ebf86880fba150650b0d98c4af3c3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T00:15:17,237 INFO [Thread-1542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1542. 2024-11-19T00:15:17,237 DEBUG [Thread-1542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:15:17,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=A 2024-11-19T00:15:17,237 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:17,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:17,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=B 2024-11-19T00:15:17,237 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:17,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:17,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=C 2024-11-19T00:15:17,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:17,238 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:15:17,246 INFO [Thread-1540 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1540. 2024-11-19T00:15:17,246 DEBUG [Thread-1540 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:15:17,246 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:17,246 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:17,246 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:15:17,246 INFO [Thread-1548 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1548. 2024-11-19T00:15:17,246 DEBUG [Thread-1548 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:15:17,246 DEBUG [Thread-1548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:17,246 DEBUG [Thread-1548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:17,246 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:15:17,247 INFO [Thread-1546 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1546. 2024-11-19T00:15:17,248 DEBUG [Thread-1546 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:15:17,248 DEBUG [Thread-1546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:17,248 DEBUG [Thread-1546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:17,248 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:15:17,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195d7ccc7b32074dea8a84874f3ae6753d_999ebf86880fba150650b0d98c4af3c3 is 50, key is test_row_0/A:col10/1731975315042/Put/seqid=0 2024-11-19T00:15:17,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742254_1430 (size=12454) 2024-11-19T00:15:17,273 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:17,277 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195d7ccc7b32074dea8a84874f3ae6753d_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195d7ccc7b32074dea8a84874f3ae6753d_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:17,278 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/c9af87e0084d4ef99c9c871d5d8423f1, store: [table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:17,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/c9af87e0084d4ef99c9c871d5d8423f1 is 175, key is test_row_0/A:col10/1731975315042/Put/seqid=0 2024-11-19T00:15:17,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742255_1431 (size=31255) 2024-11-19T00:15:17,684 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=356, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/c9af87e0084d4ef99c9c871d5d8423f1 2024-11-19T00:15:17,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/aa41bc410de04d67b32c2a3fdacfa2d4 is 50, key is test_row_0/B:col10/1731975315042/Put/seqid=0 2024-11-19T00:15:17,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742256_1432 (size=12301) 2024-11-19T00:15:18,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/aa41bc410de04d67b32c2a3fdacfa2d4 2024-11-19T00:15:18,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/1d2be706415b44a6811b2ef6d87d3726 is 50, key is test_row_0/C:col10/1731975315042/Put/seqid=0 2024-11-19T00:15:18,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742257_1433 (size=12301) 2024-11-19T00:15:18,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/1d2be706415b44a6811b2ef6d87d3726 2024-11-19T00:15:18,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/c9af87e0084d4ef99c9c871d5d8423f1 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/c9af87e0084d4ef99c9c871d5d8423f1 2024-11-19T00:15:18,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/c9af87e0084d4ef99c9c871d5d8423f1, entries=150, sequenceid=356, filesize=30.5 K 2024-11-19T00:15:18,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/aa41bc410de04d67b32c2a3fdacfa2d4 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/aa41bc410de04d67b32c2a3fdacfa2d4 2024-11-19T00:15:18,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/aa41bc410de04d67b32c2a3fdacfa2d4, entries=150, sequenceid=356, filesize=12.0 K 2024-11-19T00:15:18,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/1d2be706415b44a6811b2ef6d87d3726 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/1d2be706415b44a6811b2ef6d87d3726 2024-11-19T00:15:18,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/1d2be706415b44a6811b2ef6d87d3726, entries=150, sequenceid=356, filesize=12.0 K 2024-11-19T00:15:18,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=20.13 KB/20610 for 999ebf86880fba150650b0d98c4af3c3 in 1294ms, sequenceid=356, compaction requested=false 2024-11-19T00:15:18,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 999ebf86880fba150650b0d98c4af3c3: 2024-11-19T00:15:19,909 INFO [Thread-1544 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1544. 2024-11-19T00:15:19,909 DEBUG [Thread-1544 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:15:19,909 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:19,910 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:19,910 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:15:19,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T00:15:19,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-19T00:15:19,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-19T00:15:19,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-19T00:15:19,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-19T00:15:19,911 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-19T00:15:19,911 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T00:15:19,911 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T00:15:19,911 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1063 2024-11-19T00:15:19,911 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3187 rows 2024-11-19T00:15:19,911 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1063 2024-11-19T00:15:19,911 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3184 rows 2024-11-19T00:15:19,911 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1060 2024-11-19T00:15:19,911 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3177 rows 2024-11-19T00:15:19,911 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1066 2024-11-19T00:15:19,911 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3198 rows 2024-11-19T00:15:19,911 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1062 2024-11-19T00:15:19,911 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3186 rows 2024-11-19T00:15:19,911 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T00:15:19,911 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T00:15:19,911 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:19,911 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:19,911 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:15:19,912 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T00:15:19,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T00:15:19,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T00:15:19,916 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975319916"}]},"ts":"1731975319916"} 2024-11-19T00:15:19,918 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T00:15:19,918 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T00:15:19,918 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T00:15:19,920 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=999ebf86880fba150650b0d98c4af3c3, UNASSIGN}] 2024-11-19T00:15:19,921 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=999ebf86880fba150650b0d98c4af3c3, UNASSIGN 2024-11-19T00:15:19,921 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=999ebf86880fba150650b0d98c4af3c3, regionState=CLOSING, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:15:19,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=999ebf86880fba150650b0d98c4af3c3, UNASSIGN because future has completed 2024-11-19T00:15:19,923 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T00:15:19,923 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317}] 2024-11-19T00:15:20,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T00:15:20,075 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:20,075 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T00:15:20,075 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing 999ebf86880fba150650b0d98c4af3c3, disabling compactions & flushes 2024-11-19T00:15:20,075 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:20,075 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:20,075 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. after waiting 0 ms 2024-11-19T00:15:20,075 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:20,076 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing 999ebf86880fba150650b0d98c4af3c3 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-19T00:15:20,076 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=A 2024-11-19T00:15:20,076 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:20,076 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=B 2024-11-19T00:15:20,076 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:20,076 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 999ebf86880fba150650b0d98c4af3c3, store=C 2024-11-19T00:15:20,076 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:20,083 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a99775494c2045eaaa543e1149542927_999ebf86880fba150650b0d98c4af3c3 is 50, key is test_row_0/A:col10/1731975317244/Put/seqid=0 2024-11-19T00:15:20,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742258_1434 (size=9914) 2024-11-19T00:15:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T00:15:20,489 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:20,497 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a99775494c2045eaaa543e1149542927_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a99775494c2045eaaa543e1149542927_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:20,498 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/de7d2d4f32254477ba1a2295e612bd11, store: [table=TestAcidGuarantees family=A region=999ebf86880fba150650b0d98c4af3c3] 2024-11-19T00:15:20,499 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/de7d2d4f32254477ba1a2295e612bd11 is 175, key is test_row_0/A:col10/1731975317244/Put/seqid=0 2024-11-19T00:15:20,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742259_1435 (size=22561) 2024-11-19T00:15:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T00:15:20,903 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=363, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/de7d2d4f32254477ba1a2295e612bd11 2024-11-19T00:15:20,911 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/aaf31b78efc04662b08bf14003e629b5 is 50, key is test_row_0/B:col10/1731975317244/Put/seqid=0 2024-11-19T00:15:20,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742260_1436 (size=9857) 2024-11-19T00:15:21,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T00:15:21,317 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/aaf31b78efc04662b08bf14003e629b5 2024-11-19T00:15:21,330 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/1d6039c63f414e0d9216718363be2d96 is 50, key is test_row_0/C:col10/1731975317244/Put/seqid=0 2024-11-19T00:15:21,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742261_1437 (size=9857) 2024-11-19T00:15:21,734 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/1d6039c63f414e0d9216718363be2d96 2024-11-19T00:15:21,740 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/A/de7d2d4f32254477ba1a2295e612bd11 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/de7d2d4f32254477ba1a2295e612bd11 2024-11-19T00:15:21,744 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/de7d2d4f32254477ba1a2295e612bd11, entries=100, sequenceid=363, filesize=22.0 K 2024-11-19T00:15:21,745 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/B/aaf31b78efc04662b08bf14003e629b5 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/aaf31b78efc04662b08bf14003e629b5 2024-11-19T00:15:21,748 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/aaf31b78efc04662b08bf14003e629b5, entries=100, sequenceid=363, filesize=9.6 K 2024-11-19T00:15:21,748 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/.tmp/C/1d6039c63f414e0d9216718363be2d96 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/1d6039c63f414e0d9216718363be2d96 2024-11-19T00:15:21,751 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/1d6039c63f414e0d9216718363be2d96, entries=100, sequenceid=363, filesize=9.6 K 2024-11-19T00:15:21,752 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 999ebf86880fba150650b0d98c4af3c3 in 1677ms, sequenceid=363, compaction requested=true 2024-11-19T00:15:21,756 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/recovered.edits/366.seqid, newMaxSeqId=366, maxSeqId=4 2024-11-19T00:15:21,756 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. 2024-11-19T00:15:21,757 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for 999ebf86880fba150650b0d98c4af3c3: Waiting for close lock at 1731975320075Running coprocessor pre-close hooks at 1731975320075Disabling compacts and flushes for region at 1731975320075Disabling writes for close at 1731975320075Obtaining lock to block concurrent updates at 1731975320076 (+1 ms)Preparing flush snapshotting stores in 999ebf86880fba150650b0d98c4af3c3 at 1731975320076Finished memstore snapshotting TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731975320076Flushing stores of TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3. at 1731975320077 (+1 ms)Flushing 999ebf86880fba150650b0d98c4af3c3/A: creating writer at 1731975320077Flushing mob file 999ebf86880fba150650b0d98c4af3c3/A: appending metadata at 1731975320082 (+5 ms)Flushing mob file 999ebf86880fba150650b0d98c4af3c3/A: closing flushed file at 1731975320082Flushing 999ebf86880fba150650b0d98c4af3c3/A: appending metadata at 1731975320498 (+416 ms)Flushing 999ebf86880fba150650b0d98c4af3c3/A: closing flushed file at 1731975320498Flushing 999ebf86880fba150650b0d98c4af3c3/B: creating writer at 1731975320907 (+409 ms)Flushing 999ebf86880fba150650b0d98c4af3c3/B: appending metadata at 1731975320910 (+3 ms)Flushing 999ebf86880fba150650b0d98c4af3c3/B: closing flushed file at 1731975320910Flushing 999ebf86880fba150650b0d98c4af3c3/C: creating writer at 1731975321325 (+415 ms)Flushing 999ebf86880fba150650b0d98c4af3c3/C: appending metadata at 1731975321330 (+5 ms)Flushing 999ebf86880fba150650b0d98c4af3c3/C: closing flushed file at 1731975321330Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6dd83bd: reopening flushed file at 1731975321739 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4361a51b: reopening flushed file at 1731975321744 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67921f6e: reopening flushed file at 1731975321748 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 999ebf86880fba150650b0d98c4af3c3 in 1677ms, sequenceid=363, compaction requested=true at 1731975321752 (+4 ms)Writing region close event to WAL at 1731975321753 (+1 ms)Running coprocessor post-close hooks at 1731975321756 (+3 ms)Closed at 1731975321756 2024-11-19T00:15:21,758 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed 999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:21,759 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=999ebf86880fba150650b0d98c4af3c3, regionState=CLOSED 2024-11-19T00:15:21,760 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 because future has completed 2024-11-19T00:15:21,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-19T00:15:21,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure 999ebf86880fba150650b0d98c4af3c3, server=6b1531871d35,38567,1731975194317 in 1.8380 sec 2024-11-19T00:15:21,763 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-19T00:15:21,763 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=999ebf86880fba150650b0d98c4af3c3, UNASSIGN in 1.8420 sec 2024-11-19T00:15:21,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-19T00:15:21,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8450 sec 2024-11-19T00:15:21,765 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975321765"}]},"ts":"1731975321765"} 2024-11-19T00:15:21,766 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T00:15:21,766 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T00:15:21,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8550 sec 2024-11-19T00:15:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T00:15:22,065 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:15:22,066 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T00:15:22,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:15:22,071 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:15:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T00:15:22,072 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:15:22,075 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:22,079 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A, FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B, FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C, FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/recovered.edits] 2024-11-19T00:15:22,084 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/7353fb369c80412b994d90878c350205 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/7353fb369c80412b994d90878c350205 2024-11-19T00:15:22,086 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/c9af87e0084d4ef99c9c871d5d8423f1 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/c9af87e0084d4ef99c9c871d5d8423f1 2024-11-19T00:15:22,088 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/de7d2d4f32254477ba1a2295e612bd11 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/A/de7d2d4f32254477ba1a2295e612bd11 2024-11-19T00:15:22,091 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/aa41bc410de04d67b32c2a3fdacfa2d4 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/aa41bc410de04d67b32c2a3fdacfa2d4 2024-11-19T00:15:22,093 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/aaf31b78efc04662b08bf14003e629b5 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/aaf31b78efc04662b08bf14003e629b5 2024-11-19T00:15:22,095 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/c1b3967dd1264909956f3f41d692c081 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/B/c1b3967dd1264909956f3f41d692c081 2024-11-19T00:15:22,099 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/05306509d7494634ab33b8e158522d61 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/05306509d7494634ab33b8e158522d61 2024-11-19T00:15:22,101 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/1d2be706415b44a6811b2ef6d87d3726 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/1d2be706415b44a6811b2ef6d87d3726 2024-11-19T00:15:22,103 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/1d6039c63f414e0d9216718363be2d96 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/C/1d6039c63f414e0d9216718363be2d96 2024-11-19T00:15:22,107 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/recovered.edits/366.seqid to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3/recovered.edits/366.seqid 2024-11-19T00:15:22,107 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:22,108 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T00:15:22,108 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T00:15:22,109 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T00:15:22,112 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111914345c3f9de2421982c904a55b1c46df_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111914345c3f9de2421982c904a55b1c46df_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:22,114 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119211bd125ce6c4b29a17e9aba31ff660e_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119211bd125ce6c4b29a17e9aba31ff660e_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:22,115 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192ad49cf2ad7642d9a6315e21ceaff4a7_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192ad49cf2ad7642d9a6315e21ceaff4a7_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:22,116 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195d7ccc7b32074dea8a84874f3ae6753d_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195d7ccc7b32074dea8a84874f3ae6753d_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:22,117 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196c20cde7d6f049bb9e77410d7851f350_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196c20cde7d6f049bb9e77410d7851f350_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:22,117 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197fb229afffdb4cc283598e100a1b76b7_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197fb229afffdb4cc283598e100a1b76b7_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:22,118 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119842ad28d37504f618df56a4c207dd9b1_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119842ad28d37504f618df56a4c207dd9b1_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:22,119 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111984c9ce96306e491bbb3c3ee2a55b8b97_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111984c9ce96306e491bbb3c3ee2a55b8b97_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:22,120 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119954427da75bc4eedb7174a4f52ebc21b_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119954427da75bc4eedb7174a4f52ebc21b_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:22,121 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a99775494c2045eaaa543e1149542927_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a99775494c2045eaaa543e1149542927_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:22,122 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b8ea459abe0b46f28f409b09c2bb213c_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b8ea459abe0b46f28f409b09c2bb213c_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:22,122 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bdcd46a9db6e4ae8b77858015554092a_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bdcd46a9db6e4ae8b77858015554092a_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:22,123 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c78872cc6ab64475a81166f274896ebb_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c78872cc6ab64475a81166f274896ebb_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:22,124 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c88926d45cbe4451ab546b60481cb977_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c88926d45cbe4451ab546b60481cb977_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:22,125 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e245eece7aa84ee5991ee66ca1252ad1_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e245eece7aa84ee5991ee66ca1252ad1_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:22,126 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e75292aad4874986bfda59a392dedd1d_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e75292aad4874986bfda59a392dedd1d_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:22,126 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e96d8b98b9c8467fb9ff495e27182fb8_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e96d8b98b9c8467fb9ff495e27182fb8_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:22,127 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fdc04cd36a0a430f9d90efe15f369097_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fdc04cd36a0a430f9d90efe15f369097_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:22,128 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fecc750f092e4b6c8e9e628a7c1b26a8_999ebf86880fba150650b0d98c4af3c3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fecc750f092e4b6c8e9e628a7c1b26a8_999ebf86880fba150650b0d98c4af3c3 2024-11-19T00:15:22,128 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T00:15:22,130 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:15:22,132 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T00:15:22,133 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T00:15:22,134 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:15:22,134 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T00:15:22,134 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731975322134"}]},"ts":"9223372036854775807"} 2024-11-19T00:15:22,136 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T00:15:22,136 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 999ebf86880fba150650b0d98c4af3c3, NAME => 'TestAcidGuarantees,,1731975292554.999ebf86880fba150650b0d98c4af3c3.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T00:15:22,136 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T00:15:22,136 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731975322136"}]},"ts":"9223372036854775807"} 2024-11-19T00:15:22,138 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T00:15:22,138 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:15:22,139 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 71 msec 2024-11-19T00:15:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T00:15:22,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T00:15:22,174 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:15:22,182 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250 (was 246) - Thread LEAK? -, OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=568 (was 495) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1508 (was 1932) 2024-11-19T00:15:22,190 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=568, ProcessCount=11, AvailableMemoryMB=1507 2024-11-19T00:15:22,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T00:15:22,192 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T00:15:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T00:15:22,193 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T00:15:22,194 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:22,194 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-11-19T00:15:22,194 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T00:15:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-19T00:15:22,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742262_1438 (size=1061) 2024-11-19T00:15:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-19T00:15:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-19T00:15:22,605 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2 2024-11-19T00:15:22,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742263_1439 (size=53) 2024-11-19T00:15:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-19T00:15:23,012 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:15:23,012 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 874070de567718d211743b79a5f68965, disabling compactions & flushes 2024-11-19T00:15:23,012 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:23,012 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:23,012 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. after waiting 0 ms 2024-11-19T00:15:23,012 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:23,012 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:23,012 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 874070de567718d211743b79a5f68965: Waiting for close lock at 1731975323012Disabling compacts and flushes for region at 1731975323012Disabling writes for close at 1731975323012Writing region close event to WAL at 1731975323012Closed at 1731975323012 2024-11-19T00:15:23,014 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T00:15:23,014 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731975323014"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731975323014"}]},"ts":"1731975323014"} 2024-11-19T00:15:23,015 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T00:15:23,016 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T00:15:23,016 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975323016"}]},"ts":"1731975323016"} 2024-11-19T00:15:23,018 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T00:15:23,018 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=874070de567718d211743b79a5f68965, ASSIGN}] 2024-11-19T00:15:23,019 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=874070de567718d211743b79a5f68965, ASSIGN 2024-11-19T00:15:23,020 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=874070de567718d211743b79a5f68965, ASSIGN; state=OFFLINE, location=6b1531871d35,38567,1731975194317; forceNewPlan=false, retain=false 2024-11-19T00:15:23,171 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=874070de567718d211743b79a5f68965, regionState=OPENING, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:15:23,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=874070de567718d211743b79a5f68965, ASSIGN because future has completed 2024-11-19T00:15:23,174 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317}] 2024-11-19T00:15:23,330 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:23,330 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} 2024-11-19T00:15:23,331 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 874070de567718d211743b79a5f68965 2024-11-19T00:15:23,331 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:15:23,331 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for 874070de567718d211743b79a5f68965 2024-11-19T00:15:23,331 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for 874070de567718d211743b79a5f68965 2024-11-19T00:15:23,333 INFO [StoreOpener-874070de567718d211743b79a5f68965-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 874070de567718d211743b79a5f68965 2024-11-19T00:15:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-19T00:15:23,334 INFO [StoreOpener-874070de567718d211743b79a5f68965-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:15:23,334 INFO [StoreOpener-874070de567718d211743b79a5f68965-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 874070de567718d211743b79a5f68965 columnFamilyName A 2024-11-19T00:15:23,334 DEBUG [StoreOpener-874070de567718d211743b79a5f68965-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:23,335 INFO [StoreOpener-874070de567718d211743b79a5f68965-1 {}] regionserver.HStore(327): Store=874070de567718d211743b79a5f68965/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:15:23,335 INFO [StoreOpener-874070de567718d211743b79a5f68965-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 874070de567718d211743b79a5f68965 2024-11-19T00:15:23,336 INFO [StoreOpener-874070de567718d211743b79a5f68965-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:15:23,336 INFO [StoreOpener-874070de567718d211743b79a5f68965-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 874070de567718d211743b79a5f68965 columnFamilyName B 2024-11-19T00:15:23,336 DEBUG [StoreOpener-874070de567718d211743b79a5f68965-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:23,337 INFO [StoreOpener-874070de567718d211743b79a5f68965-1 {}] regionserver.HStore(327): Store=874070de567718d211743b79a5f68965/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:15:23,337 INFO [StoreOpener-874070de567718d211743b79a5f68965-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 874070de567718d211743b79a5f68965 2024-11-19T00:15:23,338 INFO [StoreOpener-874070de567718d211743b79a5f68965-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:15:23,338 INFO [StoreOpener-874070de567718d211743b79a5f68965-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 874070de567718d211743b79a5f68965 columnFamilyName C 2024-11-19T00:15:23,338 DEBUG [StoreOpener-874070de567718d211743b79a5f68965-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:23,338 INFO [StoreOpener-874070de567718d211743b79a5f68965-1 {}] regionserver.HStore(327): Store=874070de567718d211743b79a5f68965/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:15:23,338 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:23,339 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for 874070de567718d211743b79a5f68965 2024-11-19T00:15:23,339 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965 2024-11-19T00:15:23,339 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965 2024-11-19T00:15:23,340 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for 874070de567718d211743b79a5f68965 2024-11-19T00:15:23,340 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for 874070de567718d211743b79a5f68965 2024-11-19T00:15:23,341 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T00:15:23,342 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for 874070de567718d211743b79a5f68965 2024-11-19T00:15:23,343 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T00:15:23,343 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened 874070de567718d211743b79a5f68965; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74725154, jitterRate=0.11349156498908997}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T00:15:23,343 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 874070de567718d211743b79a5f68965 2024-11-19T00:15:23,344 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for 874070de567718d211743b79a5f68965: Running coprocessor pre-open hook at 1731975323331Writing region info on filesystem at 1731975323331Initializing all the Stores at 1731975323332 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975323332Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975323333 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975323333Cleaning up temporary data from old regions at 1731975323340 (+7 ms)Running coprocessor post-open hooks at 1731975323343 (+3 ms)Region opened successfully at 1731975323344 (+1 ms) 2024-11-19T00:15:23,345 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., pid=114, masterSystemTime=1731975323327 2024-11-19T00:15:23,347 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:23,347 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:23,347 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=874070de567718d211743b79a5f68965, regionState=OPEN, openSeqNum=2, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:15:23,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 because future has completed 2024-11-19T00:15:23,350 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-19T00:15:23,350 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure 874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 in 175 msec 2024-11-19T00:15:23,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-19T00:15:23,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=874070de567718d211743b79a5f68965, ASSIGN in 332 msec 2024-11-19T00:15:23,353 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T00:15:23,353 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975323353"}]},"ts":"1731975323353"} 2024-11-19T00:15:23,354 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T00:15:23,355 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T00:15:23,356 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1630 sec 2024-11-19T00:15:24,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-19T00:15:24,345 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:15:24,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d55bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,348 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:15:24,348 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:15:24,350 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:15:24,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:15:24,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:15:24,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b4775f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:15:24,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:15:24,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:24,354 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:15:24,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b2daa27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,356 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:15:24,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:15:24,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:15:24,358 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:15:24,359 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T00:15:24,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 6b1531871d35,33843,1731975193570 2024-11-19T00:15:24,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@76da02ec 2024-11-19T00:15:24,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T00:15:24,361 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T00:15:24,362 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,362 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:15:24,362 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:15:24,363 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:15:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:15:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:15:24,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@240b9b3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:15:24,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:15:24,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:24,365 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:15:24,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ce40163, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,366 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:15:24,366 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:15:24,367 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:15:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:15:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:15:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b816f42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:15:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:15:24,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:24,368 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:15:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e2965b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,370 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,370 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:15:24,370 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:15:24,371 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:15:24,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:15:24,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:15:24,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fb0c3a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:15:24,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:15:24,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:24,373 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:15:24,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77eb1855, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e40afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,374 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:15:24,375 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:15:24,375 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:15:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:15:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:15:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30816e30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:15:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:15:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:24,377 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:15:24,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e8324ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,378 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,378 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:15:24,379 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:15:24,379 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:15:24,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:15:24,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:15:24,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fa1ecb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:15:24,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:15:24,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:24,381 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:15:24,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6207a10e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,382 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,382 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:15:24,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:15:24,383 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:15:24,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:15:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:15:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b013a57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:15:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:15:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:24,385 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:15:24,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7860623c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,386 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,386 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:15:24,387 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:15:24,387 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:15:24,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:15:24,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:15:24,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@370bb271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:15:24,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:15:24,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:24,389 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:15:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e4f73c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,390 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:15:24,391 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:15:24,391 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:15:24,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:15:24,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:15:24,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5969a2aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:15:24,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:15:24,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:24,393 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:15:24,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63cef522, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,395 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3088fab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,395 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:15:24,395 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:15:24,396 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:15:24,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:15:24,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:15:24,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ef8c216, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:15:24,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:15:24,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:24,398 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:15:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2edd4ef4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@663bb5d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,399 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:15:24,400 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:15:24,400 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:15:24,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:15:24,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:15:24,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59a2bf70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:15:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:15:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:24,403 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:15:24,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f181e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:24,406 DEBUG [Thread-1990 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:15:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:15:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:15:24,408 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:15:24,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:15:24,415 DEBUG [Thread-1986 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:15:24,415 DEBUG [Thread-1975 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:15:24,415 DEBUG [Thread-1977 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:15:24,415 DEBUG [Thread-1979 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:15:24,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:15:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:15:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:15:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:15:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:15:24,416 DEBUG [Thread-1992 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:15:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:15:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:15:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:15:24,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:15:24,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:15:24,417 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:15:24,417 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:15:24,417 DEBUG [Thread-1981 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:15:24,418 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:15:24,418 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:15:24,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:15:24,418 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:15:24,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:15:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-19T00:15:24,418 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:15:24,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:15:24,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:15:24,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:15:24,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:15:24,421 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:15:24,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:15:24,422 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:15:24,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T00:15:24,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:15:24,423 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:15:24,423 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:15:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 874070de567718d211743b79a5f68965 2024-11-19T00:15:24,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 874070de567718d211743b79a5f68965 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T00:15:24,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=A 2024-11-19T00:15:24,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:24,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=B 2024-11-19T00:15:24,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:24,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=C 2024-11-19T00:15:24,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:24,429 DEBUG [Thread-1988 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:15:24,429 DEBUG [Thread-1983 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:15:24,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:15:24,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:15:24,432 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:15:24,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:15:24,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:15:24,434 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:15:24,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:15:24,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:15:24,438 DEBUG [Thread-1994 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:15:24,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:15:24,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:15:24,440 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:15:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2] 2024-11-19T00:15:24,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/e9863581a5794de98492d2577bedf67a is 50, key is test_row_0/A:col10/1731975324426/Put/seqid=0 2024-11-19T00:15:24,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742264_1440 (size=9657) 2024-11-19T00:15:24,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/e9863581a5794de98492d2577bedf67a 2024-11-19T00:15:24,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731975324466, max timestamp=1731975324480Num uniques 859; 2024-11-19T00:15:24,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T00:15:24,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731975324466, max timestamp=1731975324480Num uniques 859; 2024-11-19T00:15:24,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T00:15:24,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975324467, max timestamp=1731975324482Num uniques 859; 2024-11-19T00:15:24,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:15:24,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/b9f460760a8c4745b916a7ecdcbbe4d9 is 50, key is test_row_0/B:col10/1731975324426/Put/seqid=0 2024-11-19T00:15:24,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742265_1441 (size=9657) 2024-11-19T00:15:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T00:15:24,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975384529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:24,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:24,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:24,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975384542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975384543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:24,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:24,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975384544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975384545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:24,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:24,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:24,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T00:15:24,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:24,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:24,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:24,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:24,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:24,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975384653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:24,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975384664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:24,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975384664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:24,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:24,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975384664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:24,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:24,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975384664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:24,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:24,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:24,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:24,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T00:15:24,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:24,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:24,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:24,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:24,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T00:15:24,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975384874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:24,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:24,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T00:15:24,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:24,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:24,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:24,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:24,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:24,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975384883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:24,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975384883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:24,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:24,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:24,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975384884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:24,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975384884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:24,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/b9f460760a8c4745b916a7ecdcbbe4d9 2024-11-19T00:15:24,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/6bb2ad7621934df1ad9ff4d5715ecc5f is 50, key is test_row_0/C:col10/1731975324426/Put/seqid=0 2024-11-19T00:15:24,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742266_1442 (size=9657) 2024-11-19T00:15:24,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/6bb2ad7621934df1ad9ff4d5715ecc5f 2024-11-19T00:15:24,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/e9863581a5794de98492d2577bedf67a as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/e9863581a5794de98492d2577bedf67a 2024-11-19T00:15:24,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/e9863581a5794de98492d2577bedf67a, entries=100, sequenceid=11, filesize=9.4 K 2024-11-19T00:15:24,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/b9f460760a8c4745b916a7ecdcbbe4d9 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/b9f460760a8c4745b916a7ecdcbbe4d9 2024-11-19T00:15:25,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/b9f460760a8c4745b916a7ecdcbbe4d9, entries=100, sequenceid=11, filesize=9.4 K 2024-11-19T00:15:25,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/6bb2ad7621934df1ad9ff4d5715ecc5f as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/6bb2ad7621934df1ad9ff4d5715ecc5f 2024-11-19T00:15:25,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/6bb2ad7621934df1ad9ff4d5715ecc5f, entries=100, sequenceid=11, filesize=9.4 K 2024-11-19T00:15:25,019 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for 874070de567718d211743b79a5f68965 in 592ms, sequenceid=11, compaction requested=false 2024-11-19T00:15:25,019 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T00:15:25,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:25,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T00:15:25,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:25,038 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 874070de567718d211743b79a5f68965 3/3 column families, dataSize=136.42 KB heapSize=385.38 KB 2024-11-19T00:15:25,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=A 2024-11-19T00:15:25,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:25,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=B 2024-11-19T00:15:25,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:25,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=C 2024-11-19T00:15:25,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:25,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/c125e4fd9717445c91ee02898e4addbf is 50, key is test_row_0/A:col10/1731975324544/Put/seqid=0 2024-11-19T00:15:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T00:15:25,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742267_1443 (size=12001) 2024-11-19T00:15:25,067 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/c125e4fd9717445c91ee02898e4addbf 2024-11-19T00:15:25,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/eff889601947415fb5a6b41dd18600c2 is 50, key is test_row_0/B:col10/1731975324544/Put/seqid=0 2024-11-19T00:15:25,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742268_1444 (size=12001) 2024-11-19T00:15:25,089 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/eff889601947415fb5a6b41dd18600c2 2024-11-19T00:15:25,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/fa078044348a4460a6d90e0378082e5f is 50, key is test_row_0/C:col10/1731975324544/Put/seqid=0 2024-11-19T00:15:25,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742269_1445 (size=12001) 2024-11-19T00:15:25,132 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/fa078044348a4460a6d90e0378082e5f 2024-11-19T00:15:25,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/c125e4fd9717445c91ee02898e4addbf as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/c125e4fd9717445c91ee02898e4addbf 2024-11-19T00:15:25,148 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/c125e4fd9717445c91ee02898e4addbf, entries=150, sequenceid=47, filesize=11.7 K 2024-11-19T00:15:25,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/eff889601947415fb5a6b41dd18600c2 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/eff889601947415fb5a6b41dd18600c2 2024-11-19T00:15:25,154 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/eff889601947415fb5a6b41dd18600c2, entries=150, sequenceid=47, filesize=11.7 K 2024-11-19T00:15:25,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/fa078044348a4460a6d90e0378082e5f as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/fa078044348a4460a6d90e0378082e5f 2024-11-19T00:15:25,163 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/fa078044348a4460a6d90e0378082e5f, entries=150, sequenceid=47, filesize=11.7 K 2024-11-19T00:15:25,165 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~385.33 KB/394576, currentSize=0 B/0 for 874070de567718d211743b79a5f68965 in 127ms, sequenceid=47, compaction requested=false 2024-11-19T00:15:25,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:25,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:25,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-19T00:15:25,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-19T00:15:25,169 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-19T00:15:25,169 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 744 msec 2024-11-19T00:15:25,174 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 753 msec 2024-11-19T00:15:25,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 874070de567718d211743b79a5f68965 2024-11-19T00:15:25,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 874070de567718d211743b79a5f68965 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T00:15:25,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=A 2024-11-19T00:15:25,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:25,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=B 2024-11-19T00:15:25,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:25,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=C 2024-11-19T00:15:25,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:25,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/6c8a7be84aea457a9f37cd8169c8759e is 50, key is test_row_1/A:col10/1731975325210/Put/seqid=0 2024-11-19T00:15:25,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742270_1446 (size=14337) 2024-11-19T00:15:25,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/6c8a7be84aea457a9f37cd8169c8759e 2024-11-19T00:15:25,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/c578fc6d454b4cc5bd3c7d1905dd5b36 is 50, key is test_row_1/B:col10/1731975325210/Put/seqid=0 2024-11-19T00:15:25,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:25,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:25,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975385246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:25,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975385244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:25,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:25,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975385253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:25,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:25,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975325242, max timestamp=1731975325261Num uniques 859; 2024-11-19T00:15:25,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:15:25,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975325242, max timestamp=1731975325261Num uniques 859; 2024-11-19T00:15:25,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975325242, max timestamp=1731975325261Num uniques 859; 2024-11-19T00:15:25,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:15:25,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:15:25,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742271_1447 (size=9657) 2024-11-19T00:15:25,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/c578fc6d454b4cc5bd3c7d1905dd5b36 2024-11-19T00:15:25,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/fabcf6684e264d0eb968f0eb78b17af3 is 50, key is test_row_1/C:col10/1731975325210/Put/seqid=0 2024-11-19T00:15:25,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742272_1448 (size=9657) 2024-11-19T00:15:25,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/fabcf6684e264d0eb968f0eb78b17af3 2024-11-19T00:15:25,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/6c8a7be84aea457a9f37cd8169c8759e as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/6c8a7be84aea457a9f37cd8169c8759e 2024-11-19T00:15:25,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/6c8a7be84aea457a9f37cd8169c8759e, entries=200, sequenceid=59, filesize=14.0 K 2024-11-19T00:15:25,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/c578fc6d454b4cc5bd3c7d1905dd5b36 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/c578fc6d454b4cc5bd3c7d1905dd5b36 2024-11-19T00:15:25,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/c578fc6d454b4cc5bd3c7d1905dd5b36, entries=100, sequenceid=59, filesize=9.4 K 2024-11-19T00:15:25,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/fabcf6684e264d0eb968f0eb78b17af3 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/fabcf6684e264d0eb968f0eb78b17af3 2024-11-19T00:15:25,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/fabcf6684e264d0eb968f0eb78b17af3, entries=100, sequenceid=59, filesize=9.4 K 2024-11-19T00:15:25,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=100.63 KB/103050 for 874070de567718d211743b79a5f68965 in 144ms, sequenceid=59, compaction requested=true 2024-11-19T00:15:25,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:25,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:15:25,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:25,356 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:25,356 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:25,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:15:25,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:25,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:15:25,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:25,357 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31315 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:25,357 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/B is initiating minor compaction (all files) 2024-11-19T00:15:25,357 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/B in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:25,357 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/b9f460760a8c4745b916a7ecdcbbe4d9, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/eff889601947415fb5a6b41dd18600c2, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/c578fc6d454b4cc5bd3c7d1905dd5b36] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=30.6 K 2024-11-19T00:15:25,357 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35995 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:25,357 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/A is initiating minor compaction (all files) 2024-11-19T00:15:25,357 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/A in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:25,357 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/e9863581a5794de98492d2577bedf67a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/c125e4fd9717445c91ee02898e4addbf, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/6c8a7be84aea457a9f37cd8169c8759e] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=35.2 K 2024-11-19T00:15:25,358 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9863581a5794de98492d2577bedf67a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731975324426 2024-11-19T00:15:25,358 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting b9f460760a8c4745b916a7ecdcbbe4d9, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731975324426 2024-11-19T00:15:25,359 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting eff889601947415fb5a6b41dd18600c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731975324500 2024-11-19T00:15:25,359 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting c125e4fd9717445c91ee02898e4addbf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731975324500 2024-11-19T00:15:25,359 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c8a7be84aea457a9f37cd8169c8759e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731975325198 2024-11-19T00:15:25,359 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting c578fc6d454b4cc5bd3c7d1905dd5b36, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731975325210 2024-11-19T00:15:25,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 874070de567718d211743b79a5f68965 3/3 column families, dataSize=107.34 KB heapSize=305.30 KB 2024-11-19T00:15:25,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=A 2024-11-19T00:15:25,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:25,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=B 2024-11-19T00:15:25,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:25,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=C 2024-11-19T00:15:25,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:25,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 874070de567718d211743b79a5f68965 2024-11-19T00:15:25,384 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#B#compaction#372 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:25,384 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#A#compaction#373 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:25,385 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/00ee1a5126544ae0adeeb30397c2d6b8 is 50, key is test_row_0/A:col10/1731975324544/Put/seqid=0 2024-11-19T00:15:25,385 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/36d72c9860e44818a861a3922e6db24f is 50, key is test_row_0/B:col10/1731975324544/Put/seqid=0 2024-11-19T00:15:25,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/edc800c4c67948da99ba75034b14494f is 50, key is test_row_0/A:col10/1731975325362/Put/seqid=0 2024-11-19T00:15:25,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:25,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975385402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:25,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975385406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:25,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:25,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:25,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975385410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:25,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975385410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:25,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975385410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:25,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742273_1449 (size=12104) 2024-11-19T00:15:25,449 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/00ee1a5126544ae0adeeb30397c2d6b8 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/00ee1a5126544ae0adeeb30397c2d6b8 2024-11-19T00:15:25,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742274_1450 (size=12104) 2024-11-19T00:15:25,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742275_1451 (size=14341) 2024-11-19T00:15:25,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/edc800c4c67948da99ba75034b14494f 2024-11-19T00:15:25,456 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 874070de567718d211743b79a5f68965/A of 874070de567718d211743b79a5f68965 into 00ee1a5126544ae0adeeb30397c2d6b8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:25,456 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:25,456 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/36d72c9860e44818a861a3922e6db24f as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/36d72c9860e44818a861a3922e6db24f 2024-11-19T00:15:25,456 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/A, priority=13, startTime=1731975325356; duration=0sec 2024-11-19T00:15:25,456 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:25,457 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:A 2024-11-19T00:15:25,457 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:25,458 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31315 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:25,458 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/C is initiating minor compaction (all files) 2024-11-19T00:15:25,458 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/C in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:25,458 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/6bb2ad7621934df1ad9ff4d5715ecc5f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/fa078044348a4460a6d90e0378082e5f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/fabcf6684e264d0eb968f0eb78b17af3] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=30.6 K 2024-11-19T00:15:25,459 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bb2ad7621934df1ad9ff4d5715ecc5f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731975324426 2024-11-19T00:15:25,459 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa078044348a4460a6d90e0378082e5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731975324500 2024-11-19T00:15:25,460 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting fabcf6684e264d0eb968f0eb78b17af3, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731975325210 2024-11-19T00:15:25,461 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 874070de567718d211743b79a5f68965/B of 874070de567718d211743b79a5f68965 into 36d72c9860e44818a861a3922e6db24f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:25,461 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:25,461 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/B, priority=13, startTime=1731975325356; duration=0sec 2024-11-19T00:15:25,461 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:25,461 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:B 2024-11-19T00:15:25,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/9d42fb75fca74ac89809a21efd9476b6 is 50, key is test_row_0/B:col10/1731975325362/Put/seqid=0 2024-11-19T00:15:25,476 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#C#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:25,476 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/710371943140466b84e2dab677cb2a43 is 50, key is test_row_0/C:col10/1731975324544/Put/seqid=0 2024-11-19T00:15:25,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742276_1452 (size=12001) 2024-11-19T00:15:25,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742277_1453 (size=12104) 2024-11-19T00:15:25,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:25,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975385514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:25,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:25,528 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/710371943140466b84e2dab677cb2a43 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/710371943140466b84e2dab677cb2a43 2024-11-19T00:15:25,540 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 874070de567718d211743b79a5f68965/C of 874070de567718d211743b79a5f68965 into 710371943140466b84e2dab677cb2a43(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:25,540 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:25,540 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/C, priority=13, startTime=1731975325356; duration=0sec 2024-11-19T00:15:25,540 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:25,540 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:C 2024-11-19T00:15:25,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975385534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975385534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:25,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975385534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975385534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:25,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T00:15:25,564 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:15:25,564 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:15:25,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-19T00:15:25,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T00:15:25,566 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:15:25,566 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:15:25,566 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:15:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T00:15:25,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T00:15:25,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:25,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:25,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:25,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:25,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:25,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975385733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:25,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:25,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975385753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:25,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975385753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975385753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975385754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:25,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T00:15:25,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:25,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:25,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:25,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:25,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:25,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:25,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/9d42fb75fca74ac89809a21efd9476b6 2024-11-19T00:15:25,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/f0d1299641fa4ec1a0fa6568130a57ce is 50, key is test_row_0/C:col10/1731975325362/Put/seqid=0 2024-11-19T00:15:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T00:15:25,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742278_1454 (size=12001) 2024-11-19T00:15:26,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T00:15:26,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:26,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:26,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:26,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:26,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:26,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975386044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:26,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:26,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975386064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:26,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975386064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:26,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975386064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975386064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:26,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:26,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T00:15:26,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:26,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:26,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:26,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:26,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T00:15:26,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/f0d1299641fa4ec1a0fa6568130a57ce 2024-11-19T00:15:26,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/edc800c4c67948da99ba75034b14494f as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/edc800c4c67948da99ba75034b14494f 2024-11-19T00:15:26,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/edc800c4c67948da99ba75034b14494f, entries=200, sequenceid=92, filesize=14.0 K 2024-11-19T00:15:26,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/9d42fb75fca74ac89809a21efd9476b6 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/9d42fb75fca74ac89809a21efd9476b6 2024-11-19T00:15:26,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/9d42fb75fca74ac89809a21efd9476b6, entries=150, sequenceid=92, filesize=11.7 K 2024-11-19T00:15:26,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/f0d1299641fa4ec1a0fa6568130a57ce as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/f0d1299641fa4ec1a0fa6568130a57ce 2024-11-19T00:15:26,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/f0d1299641fa4ec1a0fa6568130a57ce, entries=150, sequenceid=92, filesize=11.7 K 2024-11-19T00:15:26,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~305.25 KB/312576, currentSize=73.80 KB/75570 for 874070de567718d211743b79a5f68965 in 940ms, sequenceid=92, compaction requested=false 2024-11-19T00:15:26,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:26,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T00:15:26,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:26,329 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 874070de567718d211743b79a5f68965 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T00:15:26,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=A 2024-11-19T00:15:26,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:26,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=B 2024-11-19T00:15:26,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:26,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=C 2024-11-19T00:15:26,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:26,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/5276bd7514de42faa3d537f4b82586c6 is 50, key is test_row_0/A:col10/1731975325404/Put/seqid=0 2024-11-19T00:15:26,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742279_1455 (size=12001) 2024-11-19T00:15:26,351 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/5276bd7514de42faa3d537f4b82586c6 2024-11-19T00:15:26,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/140a1e857bdf4b3495cfb72a25dd8436 is 50, key is test_row_0/B:col10/1731975325404/Put/seqid=0 2024-11-19T00:15:26,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742280_1456 (size=12001) 2024-11-19T00:15:26,370 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/140a1e857bdf4b3495cfb72a25dd8436 2024-11-19T00:15:26,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/56f37fe157a34d99945d703190f5b426 is 50, key is test_row_0/C:col10/1731975325404/Put/seqid=0 2024-11-19T00:15:26,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742281_1457 (size=12001) 2024-11-19T00:15:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 874070de567718d211743b79a5f68965 2024-11-19T00:15:26,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:26,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975386641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:26,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975386642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:26,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:26,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975386658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:26,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:26,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975386660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:26,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:26,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975386662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:26,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:26,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T00:15:26,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975386763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:26,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:26,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975386774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:26,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975386784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:26,791 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/56f37fe157a34d99945d703190f5b426 2024-11-19T00:15:26,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:26,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975386784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:26,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:26,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975386785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:26,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:26,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:26,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/5276bd7514de42faa3d537f4b82586c6 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/5276bd7514de42faa3d537f4b82586c6 2024-11-19T00:15:26,799 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/5276bd7514de42faa3d537f4b82586c6, entries=150, sequenceid=109, filesize=11.7 K 2024-11-19T00:15:26,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/140a1e857bdf4b3495cfb72a25dd8436 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/140a1e857bdf4b3495cfb72a25dd8436 2024-11-19T00:15:26,805 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/140a1e857bdf4b3495cfb72a25dd8436, entries=150, sequenceid=109, filesize=11.7 K 2024-11-19T00:15:26,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/56f37fe157a34d99945d703190f5b426 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/56f37fe157a34d99945d703190f5b426 2024-11-19T00:15:26,812 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/56f37fe157a34d99945d703190f5b426, entries=150, sequenceid=109, filesize=11.7 K 2024-11-19T00:15:26,813 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 874070de567718d211743b79a5f68965 in 484ms, sequenceid=109, compaction requested=true 2024-11-19T00:15:26,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:26,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:26,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-19T00:15:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-19T00:15:26,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-19T00:15:26,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2480 sec 2024-11-19T00:15:26,817 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.2520 sec 2024-11-19T00:15:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 874070de567718d211743b79a5f68965 2024-11-19T00:15:26,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 874070de567718d211743b79a5f68965 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T00:15:26,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=A 2024-11-19T00:15:26,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:26,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=B 2024-11-19T00:15:26,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:26,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=C 2024-11-19T00:15:26,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:26,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/7da8d94c98f848019c93542bfa609bb3 is 50, key is test_row_0/A:col10/1731975326659/Put/seqid=0 2024-11-19T00:15:26,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742282_1458 (size=14341) 2024-11-19T00:15:27,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975387042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975387045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:27,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975387047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:27,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975387048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:27,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975387049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:27,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:27,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975387164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975387164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:27,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975387164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:27,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975387173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:27,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975387174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:27,281 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T00:15:27,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975387373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:27,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/7da8d94c98f848019c93542bfa609bb3 2024-11-19T00:15:27,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975387383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:27,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975387383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:27,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/ef7f80599a8d482e859083ecdaf84fcc is 50, key is test_row_0/B:col10/1731975326659/Put/seqid=0 2024-11-19T00:15:27,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742283_1459 (size=12001) 2024-11-19T00:15:27,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975387394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:27,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975387394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:27,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:27,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:27,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975387684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:27,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:27,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975387694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:27,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975387704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:27,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:27,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T00:15:27,714 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:15:27,715 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:15:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-19T00:15:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T00:15:27,718 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:15:27,718 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:15:27,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:15:27,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975387714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:27,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:27,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:27,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975387714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:27,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:27,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:27,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/ef7f80599a8d482e859083ecdaf84fcc 2024-11-19T00:15:27,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/e038e3a2e50d4598b8479a0e54550935 is 50, key is test_row_0/C:col10/1731975326659/Put/seqid=0 2024-11-19T00:15:27,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742284_1460 (size=12001) 2024-11-19T00:15:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T00:15:27,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T00:15:27,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:27,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:27,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:27,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:27,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:28,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T00:15:28,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:28,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:28,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:28,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:28,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T00:15:28,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T00:15:28,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:28,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:28,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:28,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:28,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:28,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975388204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:28,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:28,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975388204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:28,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:28,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/e038e3a2e50d4598b8479a0e54550935 2024-11-19T00:15:28,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/7da8d94c98f848019c93542bfa609bb3 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/7da8d94c98f848019c93542bfa609bb3 2024-11-19T00:15:28,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/7da8d94c98f848019c93542bfa609bb3, entries=200, sequenceid=128, filesize=14.0 K 2024-11-19T00:15:28,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/ef7f80599a8d482e859083ecdaf84fcc as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/ef7f80599a8d482e859083ecdaf84fcc 2024-11-19T00:15:28,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/ef7f80599a8d482e859083ecdaf84fcc, entries=150, sequenceid=128, filesize=11.7 K 2024-11-19T00:15:28,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/e038e3a2e50d4598b8479a0e54550935 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/e038e3a2e50d4598b8479a0e54550935 2024-11-19T00:15:28,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975388223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:28,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:28,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:28,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:28,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/e038e3a2e50d4598b8479a0e54550935, entries=150, sequenceid=128, filesize=11.7 K 2024-11-19T00:15:28,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 874070de567718d211743b79a5f68965 in 1252ms, sequenceid=128, compaction requested=true 2024-11-19T00:15:28,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:28,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:15:28,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:28,233 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:15:28,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:15:28,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:28,233 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:15:28,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:15:28,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:28,234 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:15:28,235 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/A is initiating minor compaction (all files) 2024-11-19T00:15:28,235 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/A in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:28,235 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/00ee1a5126544ae0adeeb30397c2d6b8, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/edc800c4c67948da99ba75034b14494f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/5276bd7514de42faa3d537f4b82586c6, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/7da8d94c98f848019c93542bfa609bb3] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=51.5 K 2024-11-19T00:15:28,235 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:15:28,235 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00ee1a5126544ae0adeeb30397c2d6b8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731975324544 2024-11-19T00:15:28,235 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/B is initiating minor compaction (all files) 2024-11-19T00:15:28,235 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/B in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:28,235 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/36d72c9860e44818a861a3922e6db24f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/9d42fb75fca74ac89809a21efd9476b6, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/140a1e857bdf4b3495cfb72a25dd8436, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/ef7f80599a8d482e859083ecdaf84fcc] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=47.0 K 2024-11-19T00:15:28,235 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting edc800c4c67948da99ba75034b14494f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731975325328 2024-11-19T00:15:28,236 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 36d72c9860e44818a861a3922e6db24f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731975324544 2024-11-19T00:15:28,236 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5276bd7514de42faa3d537f4b82586c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731975325399 2024-11-19T00:15:28,236 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d42fb75fca74ac89809a21efd9476b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731975325344 2024-11-19T00:15:28,236 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 140a1e857bdf4b3495cfb72a25dd8436, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731975325399 2024-11-19T00:15:28,236 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7da8d94c98f848019c93542bfa609bb3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731975326656 2024-11-19T00:15:28,237 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting ef7f80599a8d482e859083ecdaf84fcc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731975326656 2024-11-19T00:15:28,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 874070de567718d211743b79a5f68965 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T00:15:28,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=A 2024-11-19T00:15:28,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:28,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=B 2024-11-19T00:15:28,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:28,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=C 2024-11-19T00:15:28,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:28,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 874070de567718d211743b79a5f68965 2024-11-19T00:15:28,249 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#B#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:28,249 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#A#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:28,249 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/65162ccd36264b29a1271e12790e59ed is 50, key is test_row_0/B:col10/1731975326659/Put/seqid=0 2024-11-19T00:15:28,250 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/29c5b18ac6b14c62bdbd6533f3bcaa70 is 50, key is test_row_0/A:col10/1731975326659/Put/seqid=0 2024-11-19T00:15:28,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/53a868b52b55465db530d086721e6eb3 is 50, key is test_row_0/A:col10/1731975328240/Put/seqid=0 2024-11-19T00:15:28,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742285_1461 (size=12241) 2024-11-19T00:15:28,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742286_1462 (size=12241) 2024-11-19T00:15:28,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742287_1463 (size=14541) 2024-11-19T00:15:28,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T00:15:28,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:28,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:28,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:28,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:28,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:28,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975388322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:28,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:28,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975388330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:28,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:28,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:28,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:28,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T00:15:28,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975388444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:28,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975388444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:28,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:28,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:28,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:28,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T00:15:28,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:28,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:28,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:28,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:28,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:28,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T00:15:28,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:28,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:28,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:28,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:28,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:28,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/53a868b52b55465db530d086721e6eb3 2024-11-19T00:15:28,675 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/29c5b18ac6b14c62bdbd6533f3bcaa70 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/29c5b18ac6b14c62bdbd6533f3bcaa70 2024-11-19T00:15:28,678 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/65162ccd36264b29a1271e12790e59ed as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/65162ccd36264b29a1271e12790e59ed 2024-11-19T00:15:28,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975388664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:28,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:28,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975388664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:28,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:28,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:28,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/5d9b1e6ff0b0479085f05556955aa68e is 50, key is test_row_0/B:col10/1731975328240/Put/seqid=0 2024-11-19T00:15:28,683 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 874070de567718d211743b79a5f68965/A of 874070de567718d211743b79a5f68965 into 29c5b18ac6b14c62bdbd6533f3bcaa70(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:28,683 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:28,683 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/A, priority=12, startTime=1731975328233; duration=0sec 2024-11-19T00:15:28,683 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:28,683 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:A 2024-11-19T00:15:28,683 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:15:28,684 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 874070de567718d211743b79a5f68965/B of 874070de567718d211743b79a5f68965 into 65162ccd36264b29a1271e12790e59ed(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:28,684 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:28,684 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/B, priority=12, startTime=1731975328233; duration=0sec 2024-11-19T00:15:28,684 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:28,684 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:B 2024-11-19T00:15:28,685 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:15:28,685 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/C is initiating minor compaction (all files) 2024-11-19T00:15:28,685 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/C in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:28,685 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/710371943140466b84e2dab677cb2a43, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/f0d1299641fa4ec1a0fa6568130a57ce, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/56f37fe157a34d99945d703190f5b426, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/e038e3a2e50d4598b8479a0e54550935] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=47.0 K 2024-11-19T00:15:28,685 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 710371943140466b84e2dab677cb2a43, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731975324544 2024-11-19T00:15:28,686 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0d1299641fa4ec1a0fa6568130a57ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731975325344 2024-11-19T00:15:28,686 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56f37fe157a34d99945d703190f5b426, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731975325399 2024-11-19T00:15:28,686 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting e038e3a2e50d4598b8479a0e54550935, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731975326656 2024-11-19T00:15:28,702 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#C#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:28,702 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/1f0e5660530c4b2daac5c371a7033720 is 50, key is test_row_0/C:col10/1731975326659/Put/seqid=0 2024-11-19T00:15:28,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742288_1464 (size=12151) 2024-11-19T00:15:28,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/5d9b1e6ff0b0479085f05556955aa68e 2024-11-19T00:15:28,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742289_1465 (size=12241) 2024-11-19T00:15:28,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/450fcd3f75894ab8908b4f7dde76dd50 is 50, key is test_row_0/C:col10/1731975328240/Put/seqid=0 2024-11-19T00:15:28,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742290_1466 (size=12151) 2024-11-19T00:15:28,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/450fcd3f75894ab8908b4f7dde76dd50 2024-11-19T00:15:28,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/53a868b52b55465db530d086721e6eb3 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/53a868b52b55465db530d086721e6eb3 2024-11-19T00:15:28,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/53a868b52b55465db530d086721e6eb3, entries=200, sequenceid=142, filesize=14.2 K 2024-11-19T00:15:28,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/5d9b1e6ff0b0479085f05556955aa68e as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/5d9b1e6ff0b0479085f05556955aa68e 2024-11-19T00:15:28,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/5d9b1e6ff0b0479085f05556955aa68e, entries=150, sequenceid=142, filesize=11.9 K 2024-11-19T00:15:28,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/450fcd3f75894ab8908b4f7dde76dd50 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/450fcd3f75894ab8908b4f7dde76dd50 2024-11-19T00:15:28,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/450fcd3f75894ab8908b4f7dde76dd50, entries=150, sequenceid=142, filesize=11.9 K 2024-11-19T00:15:28,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 874070de567718d211743b79a5f68965 in 533ms, sequenceid=142, compaction requested=false 2024-11-19T00:15:28,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:28,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T00:15:28,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:28,794 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 874070de567718d211743b79a5f68965 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T00:15:28,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=A 2024-11-19T00:15:28,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:28,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=B 2024-11-19T00:15:28,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:28,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=C 2024-11-19T00:15:28,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:28,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/bcd5a2b81cf24a9896e1ecf8cd72a8da is 50, key is test_row_0/A:col10/1731975328321/Put/seqid=0 2024-11-19T00:15:28,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742291_1467 (size=12151) 2024-11-19T00:15:28,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T00:15:28,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 874070de567718d211743b79a5f68965 2024-11-19T00:15:28,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:29,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975389065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:29,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:29,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:29,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:29,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975389077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:29,129 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/1f0e5660530c4b2daac5c371a7033720 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/1f0e5660530c4b2daac5c371a7033720 2024-11-19T00:15:29,139 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 874070de567718d211743b79a5f68965/C of 874070de567718d211743b79a5f68965 into 1f0e5660530c4b2daac5c371a7033720(size=12.0 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:29,139 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:29,139 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/C, priority=12, startTime=1731975328233; duration=0sec 2024-11-19T00:15:29,139 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:29,139 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:C 2024-11-19T00:15:29,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975389184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:29,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:29,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:29,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:29,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975389205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:29,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:29,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:29,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:29,236 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/bcd5a2b81cf24a9896e1ecf8cd72a8da 2024-11-19T00:15:29,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:29,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975389226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975389227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:29,238 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_2' in 874070de567718d211743b79a5f68965 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:15:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:29,238 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_1' in 874070de567718d211743b79a5f68965 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:15:29,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:29,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:29,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:29,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/e568384efd5046ed9f7a2b367d40bd05 is 50, key is test_row_0/B:col10/1731975328321/Put/seqid=0 2024-11-19T00:15:29,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975389244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:29,254 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_2' in 874070de567718d211743b79a5f68965 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:15:29,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:29,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:29,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:29,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742292_1468 (size=12151) 2024-11-19T00:15:29,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975389404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:29,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:29,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:29,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:29,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975389423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:29,664 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/e568384efd5046ed9f7a2b367d40bd05 2024-11-19T00:15:29,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/fb6f43556d7a4bed989eca7e1c11a900 is 50, key is test_row_0/C:col10/1731975328321/Put/seqid=0 2024-11-19T00:15:29,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742293_1469 (size=12151) 2024-11-19T00:15:29,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:29,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975389724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:29,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:29,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:29,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:29,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975389744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:29,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T00:15:30,074 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/fb6f43556d7a4bed989eca7e1c11a900 2024-11-19T00:15:30,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/bcd5a2b81cf24a9896e1ecf8cd72a8da as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/bcd5a2b81cf24a9896e1ecf8cd72a8da 2024-11-19T00:15:30,080 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/bcd5a2b81cf24a9896e1ecf8cd72a8da, entries=150, sequenceid=162, filesize=11.9 K 2024-11-19T00:15:30,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/e568384efd5046ed9f7a2b367d40bd05 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/e568384efd5046ed9f7a2b367d40bd05 2024-11-19T00:15:30,083 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/e568384efd5046ed9f7a2b367d40bd05, entries=150, sequenceid=162, filesize=11.9 K 2024-11-19T00:15:30,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/fb6f43556d7a4bed989eca7e1c11a900 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/fb6f43556d7a4bed989eca7e1c11a900 2024-11-19T00:15:30,086 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/fb6f43556d7a4bed989eca7e1c11a900, entries=150, sequenceid=162, filesize=11.9 K 2024-11-19T00:15:30,086 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 874070de567718d211743b79a5f68965 in 1292ms, sequenceid=162, compaction requested=true 2024-11-19T00:15:30,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:30,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:30,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-19T00:15:30,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-19T00:15:30,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-19T00:15:30,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3700 sec 2024-11-19T00:15:30,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.3750 sec 2024-11-19T00:15:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 874070de567718d211743b79a5f68965 2024-11-19T00:15:30,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 874070de567718d211743b79a5f68965 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T00:15:30,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=A 2024-11-19T00:15:30,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:30,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=B 2024-11-19T00:15:30,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:30,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=C 2024-11-19T00:15:30,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:30,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/920c84d4697f42a6937f131321e99872 is 50, key is test_row_0/A:col10/1731975329073/Put/seqid=0 2024-11-19T00:15:30,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742294_1470 (size=14541) 2024-11-19T00:15:30,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975390302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:30,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:30,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:30,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:30,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975390307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:30,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:30,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:30,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:30,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:30,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975390414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:30,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:30,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:30,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:30,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:30,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975390424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:30,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:30,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:30,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:30,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975390624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:30,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975390644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:30,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:30,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:30,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:30,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/920c84d4697f42a6937f131321e99872 2024-11-19T00:15:30,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/1066c5c8a3ce44208d39b90619abc85c is 50, key is test_row_0/B:col10/1731975329073/Put/seqid=0 2024-11-19T00:15:30,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742295_1471 (size=12151) 2024-11-19T00:15:30,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975390943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:30,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975390953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:30,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:30,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:30,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:31,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/1066c5c8a3ce44208d39b90619abc85c 2024-11-19T00:15:31,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/986481afbb26483396b8c9a9b10398ae is 50, key is test_row_0/C:col10/1731975329073/Put/seqid=0 2024-11-19T00:15:31,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742296_1472 (size=12151) 2024-11-19T00:15:31,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/986481afbb26483396b8c9a9b10398ae 2024-11-19T00:15:31,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/920c84d4697f42a6937f131321e99872 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/920c84d4697f42a6937f131321e99872 2024-11-19T00:15:31,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/920c84d4697f42a6937f131321e99872, entries=200, sequenceid=178, filesize=14.2 K 2024-11-19T00:15:31,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/1066c5c8a3ce44208d39b90619abc85c as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/1066c5c8a3ce44208d39b90619abc85c 2024-11-19T00:15:31,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/1066c5c8a3ce44208d39b90619abc85c, entries=150, sequenceid=178, filesize=11.9 K 2024-11-19T00:15:31,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/986481afbb26483396b8c9a9b10398ae as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/986481afbb26483396b8c9a9b10398ae 2024-11-19T00:15:31,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/986481afbb26483396b8c9a9b10398ae, entries=150, sequenceid=178, filesize=11.9 K 2024-11-19T00:15:31,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 874070de567718d211743b79a5f68965 in 841ms, sequenceid=178, compaction requested=true 2024-11-19T00:15:31,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:31,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:15:31,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:31,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:15:31,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:31,090 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:15:31,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:15:31,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:31,090 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:15:31,091 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:15:31,091 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53474 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:15:31,091 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/A is initiating minor compaction (all files) 2024-11-19T00:15:31,091 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/B is initiating minor compaction (all files) 2024-11-19T00:15:31,091 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/B in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:31,091 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/A in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:31,091 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/65162ccd36264b29a1271e12790e59ed, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/5d9b1e6ff0b0479085f05556955aa68e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/e568384efd5046ed9f7a2b367d40bd05, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/1066c5c8a3ce44208d39b90619abc85c] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=47.6 K 2024-11-19T00:15:31,091 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/29c5b18ac6b14c62bdbd6533f3bcaa70, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/53a868b52b55465db530d086721e6eb3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/bcd5a2b81cf24a9896e1ecf8cd72a8da, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/920c84d4697f42a6937f131321e99872] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=52.2 K 2024-11-19T00:15:31,092 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29c5b18ac6b14c62bdbd6533f3bcaa70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731975326656 2024-11-19T00:15:31,092 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 65162ccd36264b29a1271e12790e59ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731975326656 2024-11-19T00:15:31,092 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d9b1e6ff0b0479085f05556955aa68e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731975327045 2024-11-19T00:15:31,092 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53a868b52b55465db530d086721e6eb3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731975327045 2024-11-19T00:15:31,092 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcd5a2b81cf24a9896e1ecf8cd72a8da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731975328318 2024-11-19T00:15:31,092 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting e568384efd5046ed9f7a2b367d40bd05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731975328318 2024-11-19T00:15:31,092 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 920c84d4697f42a6937f131321e99872, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731975329028 2024-11-19T00:15:31,092 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 1066c5c8a3ce44208d39b90619abc85c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731975329051 2024-11-19T00:15:31,099 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#B#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:31,099 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#A#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:31,100 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/6d2a6ebc91124117ae6db7ba5a6076b5 is 50, key is test_row_0/A:col10/1731975329073/Put/seqid=0 2024-11-19T00:15:31,100 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/2c3cc9539e5e4f5b8a4d62c8e547f321 is 50, key is test_row_0/B:col10/1731975329073/Put/seqid=0 2024-11-19T00:15:31,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742298_1474 (size=12527) 2024-11-19T00:15:31,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742297_1473 (size=12527) 2024-11-19T00:15:31,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 874070de567718d211743b79a5f68965 2024-11-19T00:15:31,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 874070de567718d211743b79a5f68965 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T00:15:31,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=A 2024-11-19T00:15:31,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:31,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=B 2024-11-19T00:15:31,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:31,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=C 2024-11-19T00:15:31,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:31,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/d9bed3d42bb44967947221983c93ef11 is 50, key is test_row_0/A:col10/1731975330305/Put/seqid=0 2024-11-19T00:15:31,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742299_1475 (size=14541) 2024-11-19T00:15:31,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/d9bed3d42bb44967947221983c93ef11 2024-11-19T00:15:31,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/4e5ee7c103fe4e8b909d7eadf6114c36 is 50, key is test_row_0/B:col10/1731975330305/Put/seqid=0 2024-11-19T00:15:31,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742300_1476 (size=12151) 2024-11-19T00:15:31,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975391296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:31,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:31,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:31,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975391297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:31,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:31,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975391297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:31,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:31,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:31,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975391414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:31,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975391414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:31,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975391414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:31,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975391454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:31,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:31,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975391474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:31,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:31,508 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/6d2a6ebc91124117ae6db7ba5a6076b5 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/6d2a6ebc91124117ae6db7ba5a6076b5 2024-11-19T00:15:31,508 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/2c3cc9539e5e4f5b8a4d62c8e547f321 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/2c3cc9539e5e4f5b8a4d62c8e547f321 2024-11-19T00:15:31,512 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 874070de567718d211743b79a5f68965/B of 874070de567718d211743b79a5f68965 into 2c3cc9539e5e4f5b8a4d62c8e547f321(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:31,512 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 874070de567718d211743b79a5f68965/A of 874070de567718d211743b79a5f68965 into 6d2a6ebc91124117ae6db7ba5a6076b5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:31,512 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:31,512 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:31,512 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/B, priority=12, startTime=1731975331090; duration=0sec 2024-11-19T00:15:31,512 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/A, priority=12, startTime=1731975331089; duration=0sec 2024-11-19T00:15:31,512 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:31,512 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:B 2024-11-19T00:15:31,512 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:31,512 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:A 2024-11-19T00:15:31,512 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:15:31,513 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:15:31,513 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/C is initiating minor compaction (all files) 2024-11-19T00:15:31,513 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/C in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:31,513 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/1f0e5660530c4b2daac5c371a7033720, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/450fcd3f75894ab8908b4f7dde76dd50, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/fb6f43556d7a4bed989eca7e1c11a900, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/986481afbb26483396b8c9a9b10398ae] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=47.6 K 2024-11-19T00:15:31,513 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f0e5660530c4b2daac5c371a7033720, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731975326656 2024-11-19T00:15:31,513 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 450fcd3f75894ab8908b4f7dde76dd50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731975327045 2024-11-19T00:15:31,514 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting fb6f43556d7a4bed989eca7e1c11a900, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731975328318 2024-11-19T00:15:31,514 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 986481afbb26483396b8c9a9b10398ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731975329051 2024-11-19T00:15:31,521 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#C#compaction#400 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:31,522 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/91514e9dd19a45abb8067cbabc4e4ca5 is 50, key is test_row_0/C:col10/1731975329073/Put/seqid=0 2024-11-19T00:15:31,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742301_1477 (size=12527) 2024-11-19T00:15:31,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975391633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:31,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975391633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:31,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:31,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:31,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975391634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:31,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:31,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:31,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/4e5ee7c103fe4e8b909d7eadf6114c36 2024-11-19T00:15:31,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/e4922fccd11b4a9c96e9277ff04c4839 is 50, key is test_row_0/C:col10/1731975330305/Put/seqid=0 2024-11-19T00:15:31,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742302_1478 (size=12151) 2024-11-19T00:15:31,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T00:15:31,874 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:15:31,874 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:15:31,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-19T00:15:31,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T00:15:31,876 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:15:31,877 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:15:31,877 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:15:31,935 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/91514e9dd19a45abb8067cbabc4e4ca5 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/91514e9dd19a45abb8067cbabc4e4ca5 2024-11-19T00:15:31,939 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 874070de567718d211743b79a5f68965/C of 874070de567718d211743b79a5f68965 into 91514e9dd19a45abb8067cbabc4e4ca5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:31,939 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:31,939 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/C, priority=12, startTime=1731975331090; duration=0sec 2024-11-19T00:15:31,939 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:31,939 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:C 2024-11-19T00:15:31,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:31,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975391944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:31,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:31,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:31,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975391953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:31,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975391953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:31,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T00:15:32,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T00:15:32,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:32,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:32,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:32,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:32,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:32,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:32,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/e4922fccd11b4a9c96e9277ff04c4839 2024-11-19T00:15:32,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/d9bed3d42bb44967947221983c93ef11 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/d9bed3d42bb44967947221983c93ef11 2024-11-19T00:15:32,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/d9bed3d42bb44967947221983c93ef11, entries=200, sequenceid=196, filesize=14.2 K 2024-11-19T00:15:32,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/4e5ee7c103fe4e8b909d7eadf6114c36 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/4e5ee7c103fe4e8b909d7eadf6114c36 2024-11-19T00:15:32,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/4e5ee7c103fe4e8b909d7eadf6114c36, entries=150, sequenceid=196, filesize=11.9 K 2024-11-19T00:15:32,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/e4922fccd11b4a9c96e9277ff04c4839 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/e4922fccd11b4a9c96e9277ff04c4839 2024-11-19T00:15:32,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/e4922fccd11b4a9c96e9277ff04c4839, entries=150, sequenceid=196, filesize=11.9 K 2024-11-19T00:15:32,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 874070de567718d211743b79a5f68965 in 841ms, sequenceid=196, compaction requested=false 2024-11-19T00:15:32,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:32,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T00:15:32,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:32,182 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 874070de567718d211743b79a5f68965 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T00:15:32,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=A 2024-11-19T00:15:32,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:32,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=B 2024-11-19T00:15:32,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:32,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=C 2024-11-19T00:15:32,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:32,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/b41090609ed24a94b7bd7756bd915d4a is 50, key is test_row_0/A:col10/1731975331296/Put/seqid=0 2024-11-19T00:15:32,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742303_1479 (size=12151) 2024-11-19T00:15:32,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T00:15:32,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 874070de567718d211743b79a5f68965 2024-11-19T00:15:32,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:32,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T00:15:32,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:32,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975392496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:32,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:32,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975392496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:32,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:32,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975392499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:32,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:32,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:32,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975392509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:32,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:32,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:32,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975392509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:32,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:32,590 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/b41090609ed24a94b7bd7756bd915d4a 2024-11-19T00:15:32,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/948107b880914ee7acacc9039a6c24bf is 50, key is test_row_0/B:col10/1731975331296/Put/seqid=0 2024-11-19T00:15:32,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742304_1480 (size=12151) 2024-11-19T00:15:32,599 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/948107b880914ee7acacc9039a6c24bf 2024-11-19T00:15:32,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/319705a7e3454d97a5823eac3ed8fd15 is 50, key is test_row_0/C:col10/1731975331296/Put/seqid=0 2024-11-19T00:15:32,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742305_1481 (size=12151) 2024-11-19T00:15:32,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:32,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975392613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:32,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:32,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975392613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:32,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:32,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:32,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975392614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:32,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:32,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:32,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975392624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:32,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:32,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975392624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:32,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:32,631 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:32,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:32,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975392823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:32,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:32,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975392824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:32,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:32,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975392824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:32,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:32,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:32,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:32,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975392834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:32,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:32,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975392834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:32,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:32,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:32,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:33,010 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/319705a7e3454d97a5823eac3ed8fd15 2024-11-19T00:15:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T00:15:33,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/b41090609ed24a94b7bd7756bd915d4a as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/b41090609ed24a94b7bd7756bd915d4a 2024-11-19T00:15:33,094 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/b41090609ed24a94b7bd7756bd915d4a, entries=150, sequenceid=213, filesize=11.9 K 2024-11-19T00:15:33,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/948107b880914ee7acacc9039a6c24bf as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/948107b880914ee7acacc9039a6c24bf 2024-11-19T00:15:33,098 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/948107b880914ee7acacc9039a6c24bf, entries=150, sequenceid=213, filesize=11.9 K 2024-11-19T00:15:33,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/319705a7e3454d97a5823eac3ed8fd15 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/319705a7e3454d97a5823eac3ed8fd15 2024-11-19T00:15:33,101 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/319705a7e3454d97a5823eac3ed8fd15, entries=150, sequenceid=213, filesize=11.9 K 2024-11-19T00:15:33,102 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 874070de567718d211743b79a5f68965 in 920ms, sequenceid=213, compaction requested=true 2024-11-19T00:15:33,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:33,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:33,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-19T00:15:33,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-19T00:15:33,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-19T00:15:33,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2260 sec 2024-11-19T00:15:33,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.2320 sec 2024-11-19T00:15:33,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 874070de567718d211743b79a5f68965 2024-11-19T00:15:33,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 874070de567718d211743b79a5f68965 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T00:15:33,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=A 2024-11-19T00:15:33,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:33,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=B 2024-11-19T00:15:33,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:33,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=C 2024-11-19T00:15:33,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:33,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/c89f5fd04b8744ebbe4335258e8529cb is 50, key is test_row_0/A:col10/1731975333141/Put/seqid=0 2024-11-19T00:15:33,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742306_1482 (size=14541) 2024-11-19T00:15:33,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:33,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975393160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:33,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:33,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:33,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975393161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:33,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975393162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:33,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:33,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975393172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:33,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:33,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975393173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:33,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:33,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:33,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:33,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975393284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:33,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975393284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:33,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:33,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975393284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:33,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:33,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:33,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:33,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:33,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975393295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:33,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975393295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:33,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:33,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975393504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:33,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:33,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975393504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:33,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:33,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:33,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:33,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975393504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:33,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:33,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975393504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:33,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:33,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:33,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:33,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975393504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:33,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:33,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/c89f5fd04b8744ebbe4335258e8529cb 2024-11-19T00:15:33,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/7da7820d8b324d3a8dea4e29cacb7813 is 50, key is test_row_0/B:col10/1731975333141/Put/seqid=0 2024-11-19T00:15:33,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742307_1483 (size=12151) 2024-11-19T00:15:33,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:33,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975393823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:33,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:33,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975393824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:33,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:33,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:33,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975393824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:33,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:33,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:33,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975393824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:33,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:33,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:33,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975393824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:33,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:33,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:33,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/7da7820d8b324d3a8dea4e29cacb7813 2024-11-19T00:15:33,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/82e2225764a646df9bac9717fa98b542 is 50, key is test_row_0/C:col10/1731975333141/Put/seqid=0 2024-11-19T00:15:33,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742308_1484 (size=12151) 2024-11-19T00:15:34,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T00:15:34,024 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:15:34,024 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:15:34,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-19T00:15:34,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T00:15:34,026 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:15:34,026 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:15:34,026 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:15:34,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T00:15:34,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T00:15:34,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:34,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:34,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:34,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:34,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:34,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:34,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T00:15:34,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:34,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:34,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:34,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:34,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:34,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:34,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T00:15:34,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:34,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975394344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:34,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:34,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:34,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:34,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:34,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975394344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:34,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:34,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:34,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:34,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:34,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975394344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:34,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:34,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:34,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:34,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975394344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:34,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:34,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975394344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:34,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/82e2225764a646df9bac9717fa98b542 2024-11-19T00:15:34,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/c89f5fd04b8744ebbe4335258e8529cb as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/c89f5fd04b8744ebbe4335258e8529cb 2024-11-19T00:15:34,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/c89f5fd04b8744ebbe4335258e8529cb, entries=200, sequenceid=233, filesize=14.2 K 2024-11-19T00:15:34,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/7da7820d8b324d3a8dea4e29cacb7813 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/7da7820d8b324d3a8dea4e29cacb7813 2024-11-19T00:15:34,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/7da7820d8b324d3a8dea4e29cacb7813, entries=150, sequenceid=233, filesize=11.9 K 2024-11-19T00:15:34,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/82e2225764a646df9bac9717fa98b542 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/82e2225764a646df9bac9717fa98b542 2024-11-19T00:15:34,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/82e2225764a646df9bac9717fa98b542, entries=150, sequenceid=233, filesize=11.9 K 2024-11-19T00:15:34,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 874070de567718d211743b79a5f68965 in 1250ms, sequenceid=233, compaction requested=true 2024-11-19T00:15:34,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:34,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:15:34,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:34,392 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:15:34,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:15:34,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:34,392 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:15:34,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:15:34,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:34,393 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53760 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:15:34,393 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:15:34,393 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/B is initiating minor compaction (all files) 2024-11-19T00:15:34,393 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/A is initiating minor compaction (all files) 2024-11-19T00:15:34,393 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/B in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:34,393 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/A in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:34,393 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/6d2a6ebc91124117ae6db7ba5a6076b5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/d9bed3d42bb44967947221983c93ef11, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/b41090609ed24a94b7bd7756bd915d4a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/c89f5fd04b8744ebbe4335258e8529cb] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=52.5 K 2024-11-19T00:15:34,393 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/2c3cc9539e5e4f5b8a4d62c8e547f321, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/4e5ee7c103fe4e8b909d7eadf6114c36, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/948107b880914ee7acacc9039a6c24bf, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/7da7820d8b324d3a8dea4e29cacb7813] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=47.8 K 2024-11-19T00:15:34,393 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d2a6ebc91124117ae6db7ba5a6076b5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731975329051 2024-11-19T00:15:34,393 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c3cc9539e5e4f5b8a4d62c8e547f321, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731975329051 2024-11-19T00:15:34,394 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9bed3d42bb44967947221983c93ef11, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731975330292 2024-11-19T00:15:34,394 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e5ee7c103fe4e8b909d7eadf6114c36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731975330298 2024-11-19T00:15:34,394 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting b41090609ed24a94b7bd7756bd915d4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731975331282 2024-11-19T00:15:34,394 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 948107b880914ee7acacc9039a6c24bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731975331282 2024-11-19T00:15:34,394 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting c89f5fd04b8744ebbe4335258e8529cb, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731975332482 2024-11-19T00:15:34,394 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 7da7820d8b324d3a8dea4e29cacb7813, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731975332482 2024-11-19T00:15:34,400 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#B#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:34,401 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/330c9ada012f4a8e9ccc3e4740237142 is 50, key is test_row_0/B:col10/1731975333141/Put/seqid=0 2024-11-19T00:15:34,402 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#A#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:34,402 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/e4e8763e970643a9a431ad0baf8e2d38 is 50, key is test_row_0/A:col10/1731975333141/Put/seqid=0 2024-11-19T00:15:34,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742310_1486 (size=12663) 2024-11-19T00:15:34,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742309_1485 (size=12663) 2024-11-19T00:15:34,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T00:15:34,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:34,485 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 874070de567718d211743b79a5f68965 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T00:15:34,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=A 2024-11-19T00:15:34,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:34,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=B 2024-11-19T00:15:34,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:34,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=C 2024-11-19T00:15:34,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:34,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/5863f9cdd856493780eac15e7a561a34 is 50, key is test_row_0/A:col10/1731975333158/Put/seqid=0 2024-11-19T00:15:34,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742311_1487 (size=12151) 2024-11-19T00:15:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T00:15:34,808 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/e4e8763e970643a9a431ad0baf8e2d38 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/e4e8763e970643a9a431ad0baf8e2d38 2024-11-19T00:15:34,808 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/330c9ada012f4a8e9ccc3e4740237142 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/330c9ada012f4a8e9ccc3e4740237142 2024-11-19T00:15:34,812 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 874070de567718d211743b79a5f68965/A of 874070de567718d211743b79a5f68965 into e4e8763e970643a9a431ad0baf8e2d38(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:34,812 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:34,812 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/A, priority=12, startTime=1731975334392; duration=0sec 2024-11-19T00:15:34,812 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:34,812 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:A 2024-11-19T00:15:34,812 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:15:34,813 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 874070de567718d211743b79a5f68965/B of 874070de567718d211743b79a5f68965 into 330c9ada012f4a8e9ccc3e4740237142(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:34,813 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:34,813 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/B, priority=12, startTime=1731975334392; duration=0sec 2024-11-19T00:15:34,813 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:34,813 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:B 2024-11-19T00:15:34,813 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:15:34,813 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/C is initiating minor compaction (all files) 2024-11-19T00:15:34,813 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/C in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:34,814 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/91514e9dd19a45abb8067cbabc4e4ca5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/e4922fccd11b4a9c96e9277ff04c4839, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/319705a7e3454d97a5823eac3ed8fd15, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/82e2225764a646df9bac9717fa98b542] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=47.8 K 2024-11-19T00:15:34,814 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91514e9dd19a45abb8067cbabc4e4ca5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731975329051 2024-11-19T00:15:34,814 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4922fccd11b4a9c96e9277ff04c4839, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731975330298 2024-11-19T00:15:34,814 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 319705a7e3454d97a5823eac3ed8fd15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731975331282 2024-11-19T00:15:34,815 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82e2225764a646df9bac9717fa98b542, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731975332482 2024-11-19T00:15:34,822 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#C#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:34,823 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/363673d1efd64b56a5b948fcc89aff56 is 50, key is test_row_0/C:col10/1731975333141/Put/seqid=0 2024-11-19T00:15:34,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742312_1488 (size=12663) 2024-11-19T00:15:34,893 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/5863f9cdd856493780eac15e7a561a34 2024-11-19T00:15:34,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/216e940344fc440e975c7ad7078810e1 is 50, key is test_row_0/B:col10/1731975333158/Put/seqid=0 2024-11-19T00:15:34,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742313_1489 (size=12151) 2024-11-19T00:15:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T00:15:35,230 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/363673d1efd64b56a5b948fcc89aff56 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/363673d1efd64b56a5b948fcc89aff56 2024-11-19T00:15:35,233 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 874070de567718d211743b79a5f68965/C of 874070de567718d211743b79a5f68965 into 363673d1efd64b56a5b948fcc89aff56(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:35,233 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:35,233 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/C, priority=12, startTime=1731975334392; duration=0sec 2024-11-19T00:15:35,233 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:35,233 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:C 2024-11-19T00:15:35,304 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/216e940344fc440e975c7ad7078810e1 2024-11-19T00:15:35,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/d0d415c53d384cfabdd817a190634154 is 50, key is test_row_0/C:col10/1731975333158/Put/seqid=0 2024-11-19T00:15:35,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742314_1490 (size=12151) 2024-11-19T00:15:35,340 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/d0d415c53d384cfabdd817a190634154 2024-11-19T00:15:35,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/5863f9cdd856493780eac15e7a561a34 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/5863f9cdd856493780eac15e7a561a34 2024-11-19T00:15:35,348 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/5863f9cdd856493780eac15e7a561a34, entries=150, sequenceid=245, filesize=11.9 K 2024-11-19T00:15:35,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/216e940344fc440e975c7ad7078810e1 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/216e940344fc440e975c7ad7078810e1 2024-11-19T00:15:35,352 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/216e940344fc440e975c7ad7078810e1, entries=150, sequenceid=245, filesize=11.9 K 2024-11-19T00:15:35,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/d0d415c53d384cfabdd817a190634154 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/d0d415c53d384cfabdd817a190634154 2024-11-19T00:15:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 874070de567718d211743b79a5f68965 2024-11-19T00:15:35,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:35,369 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/d0d415c53d384cfabdd817a190634154, entries=150, sequenceid=245, filesize=11.9 K 2024-11-19T00:15:35,370 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=26.84 KB/27480 for 874070de567718d211743b79a5f68965 in 885ms, sequenceid=245, compaction requested=false 2024-11-19T00:15:35,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:35,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:35,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-19T00:15:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-19T00:15:35,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 874070de567718d211743b79a5f68965 2024-11-19T00:15:35,372 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 874070de567718d211743b79a5f68965 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T00:15:35,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=A 2024-11-19T00:15:35,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:35,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=B 2024-11-19T00:15:35,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:35,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=C 2024-11-19T00:15:35,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:35,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-19T00:15:35,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3450 sec 2024-11-19T00:15:35,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.3500 sec 2024-11-19T00:15:35,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/184b874dd86b46a2892e512dd34cdc81 is 50, key is test_row_0/A:col10/1731975335371/Put/seqid=0 2024-11-19T00:15:35,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742315_1491 (size=14641) 2024-11-19T00:15:35,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731975335398, max timestamp=1731975335423Num uniques 859; 2024-11-19T00:15:35,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T00:15:35,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731975335398, max timestamp=1731975335423Num uniques 859; 2024-11-19T00:15:35,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731975335398, max timestamp=1731975335423Num uniques 859; 2024-11-19T00:15:35,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T00:15:35,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T00:15:35,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:35,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975395453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:35,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:35,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975395456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:35,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975395466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:35,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:35,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975395467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:35,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:35,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975395469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:35,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:35,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975395573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:35,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975395573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:35,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:35,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:35,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975395594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:35,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:35,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975395594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:35,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:35,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:35,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975395594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:35,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:35,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:35,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/184b874dd86b46a2892e512dd34cdc81 2024-11-19T00:15:35,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/5ee5e82355364aacb02bdfad0cfc0318 is 50, key is test_row_0/B:col10/1731975335371/Put/seqid=0 2024-11-19T00:15:35,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742316_1492 (size=12201) 2024-11-19T00:15:35,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975395784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:35,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:35,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975395784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:35,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:35,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:35,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975395814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:35,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975395814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:35,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:35,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:35,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975395814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:35,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:35,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:35,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:36,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:36,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975396104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975396104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:36,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:36,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:36,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975396134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:36,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:36,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975396134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:36,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:36,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:36,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975396134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:36,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:36,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:36,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T00:15:36,174 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:15:36,174 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:15:36,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-19T00:15:36,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T00:15:36,176 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:15:36,177 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:15:36,177 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:15:36,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/5ee5e82355364aacb02bdfad0cfc0318 2024-11-19T00:15:36,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/ec0dbef8debb4d5ebc06c8dc96c105b8 is 50, key is test_row_0/C:col10/1731975335371/Put/seqid=0 2024-11-19T00:15:36,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742317_1493 (size=12201) 2024-11-19T00:15:36,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T00:15:36,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T00:15:36,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:36,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:36,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:36,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:36,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:36,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:36,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T00:15:36,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:36,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:36,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:36,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:36,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T00:15:36,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/ec0dbef8debb4d5ebc06c8dc96c105b8 2024-11-19T00:15:36,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/184b874dd86b46a2892e512dd34cdc81 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/184b874dd86b46a2892e512dd34cdc81 2024-11-19T00:15:36,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/184b874dd86b46a2892e512dd34cdc81, entries=200, sequenceid=258, filesize=14.3 K 2024-11-19T00:15:36,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/5ee5e82355364aacb02bdfad0cfc0318 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/5ee5e82355364aacb02bdfad0cfc0318 2024-11-19T00:15:36,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/5ee5e82355364aacb02bdfad0cfc0318, entries=150, sequenceid=258, filesize=11.9 K 2024-11-19T00:15:36,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/ec0dbef8debb4d5ebc06c8dc96c105b8 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/ec0dbef8debb4d5ebc06c8dc96c105b8 2024-11-19T00:15:36,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/ec0dbef8debb4d5ebc06c8dc96c105b8, entries=150, sequenceid=258, filesize=11.9 K 2024-11-19T00:15:36,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 874070de567718d211743b79a5f68965 in 1245ms, sequenceid=258, compaction requested=true 2024-11-19T00:15:36,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:36,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:15:36,617 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:36,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:36,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:15:36,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:36,618 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:36,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:15:36,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:36,618 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:36,618 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:36,618 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/A is initiating minor compaction (all files) 2024-11-19T00:15:36,618 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/B is initiating minor compaction (all files) 2024-11-19T00:15:36,618 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/A in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:36,618 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/B in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:36,618 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/e4e8763e970643a9a431ad0baf8e2d38, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/5863f9cdd856493780eac15e7a561a34, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/184b874dd86b46a2892e512dd34cdc81] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=38.5 K 2024-11-19T00:15:36,618 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/330c9ada012f4a8e9ccc3e4740237142, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/216e940344fc440e975c7ad7078810e1, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/5ee5e82355364aacb02bdfad0cfc0318] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=36.1 K 2024-11-19T00:15:36,619 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 330c9ada012f4a8e9ccc3e4740237142, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731975332482 2024-11-19T00:15:36,620 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4e8763e970643a9a431ad0baf8e2d38, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731975332482 2024-11-19T00:15:36,620 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 216e940344fc440e975c7ad7078810e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731975333158 2024-11-19T00:15:36,620 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5863f9cdd856493780eac15e7a561a34, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731975333158 2024-11-19T00:15:36,621 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 184b874dd86b46a2892e512dd34cdc81, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731975335368 2024-11-19T00:15:36,621 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ee5e82355364aacb02bdfad0cfc0318, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731975335368 2024-11-19T00:15:36,627 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#B#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:36,628 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#A#compaction#418 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:36,628 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/80c6ac8d30c64b15bde6b1af139390ad is 50, key is test_row_0/B:col10/1731975335371/Put/seqid=0 2024-11-19T00:15:36,628 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/ccb58ded1b2c458e9093dc20d583a2fb is 50, key is test_row_0/A:col10/1731975335371/Put/seqid=0 2024-11-19T00:15:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 874070de567718d211743b79a5f68965 2024-11-19T00:15:36,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 874070de567718d211743b79a5f68965 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-19T00:15:36,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T00:15:36,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:36,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:36,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:36,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:36,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:36,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=A 2024-11-19T00:15:36,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:36,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=B 2024-11-19T00:15:36,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:36,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=C 2024-11-19T00:15:36,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:36,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742319_1495 (size=12815) 2024-11-19T00:15:36,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/08b5824dbccf4689ab22f9f40f96d2ab is 50, key is test_row_0/A:col10/1731975335466/Put/seqid=0 2024-11-19T00:15:36,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742318_1494 (size=12815) 2024-11-19T00:15:36,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742320_1496 (size=17181) 2024-11-19T00:15:36,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/08b5824dbccf4689ab22f9f40f96d2ab 2024-11-19T00:15:36,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/1d14d4a0192643c891add1de5b68db5d is 50, key is test_row_0/B:col10/1731975335466/Put/seqid=0 2024-11-19T00:15:36,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742321_1497 (size=12301) 2024-11-19T00:15:36,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975396656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:36,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:36,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975396663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:36,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975396663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:36,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975396670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:36,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975396671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:36,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975396773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:36,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:36,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T00:15:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:36,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:36,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T00:15:36,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975396793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975396794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:36,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T00:15:36,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:36,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:36,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:36,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:36,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:36,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:36,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:36,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975396983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:37,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:37,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975397014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:37,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:37,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:37,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975397014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:37,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:37,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:37,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:37,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:37,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:37,043 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/ccb58ded1b2c458e9093dc20d583a2fb as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/ccb58ded1b2c458e9093dc20d583a2fb 2024-11-19T00:15:37,046 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 874070de567718d211743b79a5f68965/A of 874070de567718d211743b79a5f68965 into ccb58ded1b2c458e9093dc20d583a2fb(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:37,046 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:37,046 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/A, priority=13, startTime=1731975336617; duration=0sec 2024-11-19T00:15:37,047 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:37,047 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:A 2024-11-19T00:15:37,047 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:37,047 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:37,047 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/C is initiating minor compaction (all files) 2024-11-19T00:15:37,047 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/C in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:37,048 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/363673d1efd64b56a5b948fcc89aff56, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/d0d415c53d384cfabdd817a190634154, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/ec0dbef8debb4d5ebc06c8dc96c105b8] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=36.1 K 2024-11-19T00:15:37,048 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 363673d1efd64b56a5b948fcc89aff56, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731975332482 2024-11-19T00:15:37,048 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0d415c53d384cfabdd817a190634154, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731975333158 2024-11-19T00:15:37,048 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec0dbef8debb4d5ebc06c8dc96c105b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731975335368 2024-11-19T00:15:37,051 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/80c6ac8d30c64b15bde6b1af139390ad as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/80c6ac8d30c64b15bde6b1af139390ad 2024-11-19T00:15:37,055 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 874070de567718d211743b79a5f68965/B of 874070de567718d211743b79a5f68965 into 80c6ac8d30c64b15bde6b1af139390ad(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:37,055 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:37,055 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/B, priority=13, startTime=1731975336617; duration=0sec 2024-11-19T00:15:37,055 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:37,055 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:B 2024-11-19T00:15:37,056 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#C#compaction#421 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:37,056 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/95d5effa99c24d4fa45b685fa844a5f6 is 50, key is test_row_0/C:col10/1731975335371/Put/seqid=0 2024-11-19T00:15:37,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/1d14d4a0192643c891add1de5b68db5d 2024-11-19T00:15:37,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742322_1498 (size=12815) 2024-11-19T00:15:37,069 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/95d5effa99c24d4fa45b685fa844a5f6 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/95d5effa99c24d4fa45b685fa844a5f6 2024-11-19T00:15:37,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/f91ff4b6c0524d999f6373ecee30b17e is 50, key is test_row_0/C:col10/1731975335466/Put/seqid=0 2024-11-19T00:15:37,075 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 874070de567718d211743b79a5f68965/C of 874070de567718d211743b79a5f68965 into 95d5effa99c24d4fa45b685fa844a5f6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:37,075 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:37,075 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/C, priority=13, startTime=1731975336618; duration=0sec 2024-11-19T00:15:37,075 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:37,075 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:C 2024-11-19T00:15:37,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742323_1499 (size=12301) 2024-11-19T00:15:37,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T00:15:37,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:37,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:37,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:37,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:37,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:37,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:37,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T00:15:37,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:37,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:37,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:37,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:37,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:37,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975397294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:37,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:37,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:37,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T00:15:37,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975397333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:37,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975397333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:37,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T00:15:37,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:37,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:37,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:37,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:37,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:37,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/f91ff4b6c0524d999f6373ecee30b17e 2024-11-19T00:15:37,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/08b5824dbccf4689ab22f9f40f96d2ab as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/08b5824dbccf4689ab22f9f40f96d2ab 2024-11-19T00:15:37,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/08b5824dbccf4689ab22f9f40f96d2ab, entries=250, sequenceid=295, filesize=16.8 K 2024-11-19T00:15:37,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/1d14d4a0192643c891add1de5b68db5d as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/1d14d4a0192643c891add1de5b68db5d 2024-11-19T00:15:37,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/1d14d4a0192643c891add1de5b68db5d, entries=150, sequenceid=295, filesize=12.0 K 2024-11-19T00:15:37,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/f91ff4b6c0524d999f6373ecee30b17e as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/f91ff4b6c0524d999f6373ecee30b17e 2024-11-19T00:15:37,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/f91ff4b6c0524d999f6373ecee30b17e, entries=150, sequenceid=295, filesize=12.0 K 2024-11-19T00:15:37,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=40.25 KB/41220 for 874070de567718d211743b79a5f68965 in 872ms, sequenceid=295, compaction requested=false 2024-11-19T00:15:37,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:37,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T00:15:37,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:37,551 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 874070de567718d211743b79a5f68965 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-19T00:15:37,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=A 2024-11-19T00:15:37,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:37,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=B 2024-11-19T00:15:37,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:37,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=C 2024-11-19T00:15:37,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:37,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/cffd7e382d9d46afa91dba1e53e43181 is 50, key is test_row_0/A:col10/1731975336668/Put/seqid=0 2024-11-19T00:15:37,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742324_1500 (size=12301) 2024-11-19T00:15:37,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 874070de567718d211743b79a5f68965 2024-11-19T00:15:37,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:37,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975337723, max timestamp=1731975337747Num uniques 859; 2024-11-19T00:15:37,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:15:37,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975337723, max timestamp=1731975337747Num uniques 859; 2024-11-19T00:15:37,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:15:37,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975337723, max timestamp=1731975337747Num uniques 859; 2024-11-19T00:15:37,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:15:37,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:37,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975397813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:37,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:37,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975397815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:37,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:37,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:37,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:37,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975397817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:37,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975397854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:37,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975397854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:37,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975397934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:37,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:37,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:37,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975397934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:37,974 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/cffd7e382d9d46afa91dba1e53e43181 2024-11-19T00:15:37,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/78616000ac7345889894f8c1d3ecba67 is 50, key is test_row_0/B:col10/1731975336668/Put/seqid=0 2024-11-19T00:15:37,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742325_1501 (size=12301) 2024-11-19T00:15:37,982 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/78616000ac7345889894f8c1d3ecba67 2024-11-19T00:15:37,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/da93a31547da4f20aa28dc1ea515e9b3 is 50, key is test_row_0/C:col10/1731975336668/Put/seqid=0 2024-11-19T00:15:37,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742326_1502 (size=12301) 2024-11-19T00:15:38,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975398154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:38,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975398154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:38,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:38,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:38,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:38,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:38,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T00:15:38,392 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/da93a31547da4f20aa28dc1ea515e9b3 2024-11-19T00:15:38,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/cffd7e382d9d46afa91dba1e53e43181 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/cffd7e382d9d46afa91dba1e53e43181 2024-11-19T00:15:38,399 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/cffd7e382d9d46afa91dba1e53e43181, entries=150, sequenceid=307, filesize=12.0 K 2024-11-19T00:15:38,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/78616000ac7345889894f8c1d3ecba67 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/78616000ac7345889894f8c1d3ecba67 2024-11-19T00:15:38,403 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/78616000ac7345889894f8c1d3ecba67, entries=150, sequenceid=307, filesize=12.0 K 2024-11-19T00:15:38,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/da93a31547da4f20aa28dc1ea515e9b3 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/da93a31547da4f20aa28dc1ea515e9b3 2024-11-19T00:15:38,406 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/da93a31547da4f20aa28dc1ea515e9b3, entries=150, sequenceid=307, filesize=12.0 K 2024-11-19T00:15:38,407 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 874070de567718d211743b79a5f68965 in 856ms, sequenceid=307, compaction requested=true 2024-11-19T00:15:38,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:38,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:38,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-19T00:15:38,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-19T00:15:38,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-19T00:15:38,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2310 sec 2024-11-19T00:15:38,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.2350 sec 2024-11-19T00:15:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 874070de567718d211743b79a5f68965 2024-11-19T00:15:38,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 874070de567718d211743b79a5f68965 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-19T00:15:38,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=A 2024-11-19T00:15:38,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:15:38,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=B 2024-11-19T00:15:38,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:15:38,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=C 2024-11-19T00:15:38,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:15:38,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/970cf5fcf4eb4cb9b7cb3eeec482057b is 50, key is test_row_0/A:col10/1731975337816/Put/seqid=0 2024-11-19T00:15:38,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:38,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975398497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:38,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:38,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975398498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:38,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:38,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:38,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:38,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:38,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:38,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:38,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975398613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:38,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:38,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:38,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:38,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975398613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:38,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:38,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:38,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:38,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742327_1503 (size=14741) 2024-11-19T00:15:38,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:38,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975398824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:38,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:38,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975398824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:38,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:38,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:38,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:38,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:38,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:38,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:38,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975398833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:38,836 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_2' in 874070de567718d211743b79a5f68965 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:15:38,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:38,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:38,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:38,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975398874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:38,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975398874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:38,879 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_0' in 874070de567718d211743b79a5f68965 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:15:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:38,879 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_1' in 874070de567718d211743b79a5f68965 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:15:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:38,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:38,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:39,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/970cf5fcf4eb4cb9b7cb3eeec482057b 2024-11-19T00:15:39,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/11cafdb53bd34a2b9ad2fd33fb546d25 is 50, key is test_row_0/B:col10/1731975337816/Put/seqid=0 2024-11-19T00:15:39,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742328_1504 (size=12301) 2024-11-19T00:15:39,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/11cafdb53bd34a2b9ad2fd33fb546d25 2024-11-19T00:15:39,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/8e9f2e7cc58a40e095322e604920dc06 is 50, key is test_row_0/C:col10/1731975337816/Put/seqid=0 2024-11-19T00:15:39,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742329_1505 (size=12301) 2024-11-19T00:15:39,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975399144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:39,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975399144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:39,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:39,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:39,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:39,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:39,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:39,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:39,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/8e9f2e7cc58a40e095322e604920dc06 2024-11-19T00:15:39,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/970cf5fcf4eb4cb9b7cb3eeec482057b as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/970cf5fcf4eb4cb9b7cb3eeec482057b 2024-11-19T00:15:39,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/970cf5fcf4eb4cb9b7cb3eeec482057b, entries=200, sequenceid=345, filesize=14.4 K 2024-11-19T00:15:39,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/11cafdb53bd34a2b9ad2fd33fb546d25 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/11cafdb53bd34a2b9ad2fd33fb546d25 2024-11-19T00:15:39,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/11cafdb53bd34a2b9ad2fd33fb546d25, entries=150, sequenceid=345, filesize=12.0 K 2024-11-19T00:15:39,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/8e9f2e7cc58a40e095322e604920dc06 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/8e9f2e7cc58a40e095322e604920dc06 2024-11-19T00:15:39,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/8e9f2e7cc58a40e095322e604920dc06, entries=150, sequenceid=345, filesize=12.0 K 2024-11-19T00:15:39,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 874070de567718d211743b79a5f68965 in 1098ms, sequenceid=345, compaction requested=true 2024-11-19T00:15:39,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:39,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:15:39,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:39,580 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:15:39,580 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:15:39,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:15:39,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:39,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:15:39,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:39,585 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57038 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:15:39,585 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/A is initiating minor compaction (all files) 2024-11-19T00:15:39,586 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/A in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:39,586 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/ccb58ded1b2c458e9093dc20d583a2fb, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/08b5824dbccf4689ab22f9f40f96d2ab, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/cffd7e382d9d46afa91dba1e53e43181, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/970cf5fcf4eb4cb9b7cb3eeec482057b] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=55.7 K 2024-11-19T00:15:39,586 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49718 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:15:39,587 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/B is initiating minor compaction (all files) 2024-11-19T00:15:39,587 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/B in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:39,587 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/80c6ac8d30c64b15bde6b1af139390ad, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/1d14d4a0192643c891add1de5b68db5d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/78616000ac7345889894f8c1d3ecba67, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/11cafdb53bd34a2b9ad2fd33fb546d25] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=48.6 K 2024-11-19T00:15:39,587 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccb58ded1b2c458e9093dc20d583a2fb, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731975335368 2024-11-19T00:15:39,588 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 80c6ac8d30c64b15bde6b1af139390ad, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731975335368 2024-11-19T00:15:39,588 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08b5824dbccf4689ab22f9f40f96d2ab, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731975335450 2024-11-19T00:15:39,588 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d14d4a0192643c891add1de5b68db5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731975335466 2024-11-19T00:15:39,588 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting cffd7e382d9d46afa91dba1e53e43181, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731975336649 2024-11-19T00:15:39,589 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 78616000ac7345889894f8c1d3ecba67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731975336649 2024-11-19T00:15:39,589 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 970cf5fcf4eb4cb9b7cb3eeec482057b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731975337797 2024-11-19T00:15:39,589 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 11cafdb53bd34a2b9ad2fd33fb546d25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731975337814 2024-11-19T00:15:39,624 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#B#compaction#429 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:39,627 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/e0085f70b651473d927abf72f99822f4 is 50, key is test_row_0/B:col10/1731975337816/Put/seqid=0 2024-11-19T00:15:39,637 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#A#compaction#430 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:39,637 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/5da83c83bcfb4853bee47f275d9cd3f8 is 50, key is test_row_0/A:col10/1731975337816/Put/seqid=0 2024-11-19T00:15:39,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742330_1506 (size=13051) 2024-11-19T00:15:39,670 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/e0085f70b651473d927abf72f99822f4 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/e0085f70b651473d927abf72f99822f4 2024-11-19T00:15:39,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742331_1507 (size=13051) 2024-11-19T00:15:39,679 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 874070de567718d211743b79a5f68965/B of 874070de567718d211743b79a5f68965 into e0085f70b651473d927abf72f99822f4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:39,679 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:39,679 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/B, priority=12, startTime=1731975339580; duration=0sec 2024-11-19T00:15:39,679 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:39,679 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:B 2024-11-19T00:15:39,679 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:15:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 874070de567718d211743b79a5f68965 2024-11-19T00:15:39,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 874070de567718d211743b79a5f68965 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T00:15:39,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=A 2024-11-19T00:15:39,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:39,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=B 2024-11-19T00:15:39,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:39,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=C 2024-11-19T00:15:39,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:39,685 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49718 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:15:39,685 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/C is initiating minor compaction (all files) 2024-11-19T00:15:39,685 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/C in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:39,685 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/95d5effa99c24d4fa45b685fa844a5f6, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/f91ff4b6c0524d999f6373ecee30b17e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/da93a31547da4f20aa28dc1ea515e9b3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/8e9f2e7cc58a40e095322e604920dc06] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=48.6 K 2024-11-19T00:15:39,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/ccd2b84bcc7a4a7fa2e6a9f8761b322f is 50, key is test_row_0/A:col10/1731975339678/Put/seqid=0 2024-11-19T00:15:39,686 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 95d5effa99c24d4fa45b685fa844a5f6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731975335368 2024-11-19T00:15:39,687 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting f91ff4b6c0524d999f6373ecee30b17e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731975335466 2024-11-19T00:15:39,687 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting da93a31547da4f20aa28dc1ea515e9b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731975336649 2024-11-19T00:15:39,687 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e9f2e7cc58a40e095322e604920dc06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731975337814 2024-11-19T00:15:39,721 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#C#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:39,721 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/6cb229f8c53449d5a0b85e931fbd1646 is 50, key is test_row_0/C:col10/1731975337816/Put/seqid=0 2024-11-19T00:15:39,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742332_1508 (size=14741) 2024-11-19T00:15:39,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/ccd2b84bcc7a4a7fa2e6a9f8761b322f 2024-11-19T00:15:39,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/d001084ff2df4256afeb16ea15c858be is 50, key is test_row_0/B:col10/1731975339678/Put/seqid=0 2024-11-19T00:15:39,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742333_1509 (size=13051) 2024-11-19T00:15:39,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742334_1510 (size=12301) 2024-11-19T00:15:39,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975339764, max timestamp=1731975339810Num uniques 859; 2024-11-19T00:15:39,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975339764, max timestamp=1731975339810Num uniques 859; 2024-11-19T00:15:39,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975339764, max timestamp=1731975339810Num uniques 859; 2024-11-19T00:15:39,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:15:39,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:15:39,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:15:39,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975399959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:39,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:39,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:39,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:39,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975399977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:39,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:39,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:39,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:40,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975400084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:40,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:40,090 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/5da83c83bcfb4853bee47f275d9cd3f8 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/5da83c83bcfb4853bee47f275d9cd3f8 2024-11-19T00:15:40,095 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 874070de567718d211743b79a5f68965/A of 874070de567718d211743b79a5f68965 into 5da83c83bcfb4853bee47f275d9cd3f8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:40,095 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:40,095 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/A, priority=12, startTime=1731975339580; duration=0sec 2024-11-19T00:15:40,096 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:40,096 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:A 2024-11-19T00:15:40,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975400094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:40,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:40,156 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/6cb229f8c53449d5a0b85e931fbd1646 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/6cb229f8c53449d5a0b85e931fbd1646 2024-11-19T00:15:40,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/d001084ff2df4256afeb16ea15c858be 2024-11-19T00:15:40,162 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 874070de567718d211743b79a5f68965/C of 874070de567718d211743b79a5f68965 into 6cb229f8c53449d5a0b85e931fbd1646(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:40,162 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:40,162 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/C, priority=12, startTime=1731975339580; duration=0sec 2024-11-19T00:15:40,163 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:40,163 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:C 2024-11-19T00:15:40,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/7eb80b2cceb4486893cf5f4cb896bf58 is 50, key is test_row_0/C:col10/1731975339678/Put/seqid=0 2024-11-19T00:15:40,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742335_1511 (size=12301) 2024-11-19T00:15:40,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:40,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975400294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:40,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:40,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975400314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T00:15:40,345 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:15:40,345 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:15:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-19T00:15:40,350 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:15:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T00:15:40,351 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:15:40,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:15:40,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T00:15:40,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T00:15:40,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:40,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:40,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:40,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:40,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:40,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:40,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/7eb80b2cceb4486893cf5f4cb896bf58 2024-11-19T00:15:40,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/ccd2b84bcc7a4a7fa2e6a9f8761b322f as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/ccd2b84bcc7a4a7fa2e6a9f8761b322f 2024-11-19T00:15:40,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/ccd2b84bcc7a4a7fa2e6a9f8761b322f, entries=200, sequenceid=356, filesize=14.4 K 2024-11-19T00:15:40,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/d001084ff2df4256afeb16ea15c858be as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/d001084ff2df4256afeb16ea15c858be 2024-11-19T00:15:40,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/d001084ff2df4256afeb16ea15c858be, entries=150, sequenceid=356, filesize=12.0 K 2024-11-19T00:15:40,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/7eb80b2cceb4486893cf5f4cb896bf58 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/7eb80b2cceb4486893cf5f4cb896bf58 2024-11-19T00:15:40,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/7eb80b2cceb4486893cf5f4cb896bf58, entries=150, sequenceid=356, filesize=12.0 K 2024-11-19T00:15:40,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 874070de567718d211743b79a5f68965 in 907ms, sequenceid=356, compaction requested=false 2024-11-19T00:15:40,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:40,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 874070de567718d211743b79a5f68965 2024-11-19T00:15:40,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 874070de567718d211743b79a5f68965 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-19T00:15:40,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=A 2024-11-19T00:15:40,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:40,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=B 2024-11-19T00:15:40,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:40,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=C 2024-11-19T00:15:40,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:40,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/a0c7714b738f41bf8ec1f764ee632a76 is 50, key is test_row_0/A:col10/1731975339975/Put/seqid=0 2024-11-19T00:15:40,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742336_1512 (size=14741) 2024-11-19T00:15:40,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/a0c7714b738f41bf8ec1f764ee632a76 2024-11-19T00:15:40,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/fecaf7220f99456ab19f660118d4c981 is 50, key is test_row_0/B:col10/1731975339975/Put/seqid=0 2024-11-19T00:15:40,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742337_1513 (size=12301) 2024-11-19T00:15:40,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975400647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:40,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/fecaf7220f99456ab19f660118d4c981 2024-11-19T00:15:40,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T00:15:40,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:40,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:40,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:40,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:40,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:40,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:40,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/caca06085414442d9ccacaf5fb3c15e6 is 50, key is test_row_0/C:col10/1731975339975/Put/seqid=0 2024-11-19T00:15:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T00:15:40,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742338_1514 (size=12301) 2024-11-19T00:15:40,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975400653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:40,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:40,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975400763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:40,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:40,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:40,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975400773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:40,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T00:15:40,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:40,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:40,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:40,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:40,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:40,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60388 deadline: 1731975400853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:40,859 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_2' in 874070de567718d211743b79a5f68965 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:15:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:40,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:40,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60338 deadline: 1731975400894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:40,900 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_1' in 874070de567718d211743b79a5f68965 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:15:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:40,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1731975400904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:40,914 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 6b1531871d35,38567,1731975194317 for 'test_row_0' in 874070de567718d211743b79a5f68965 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T00:15:40,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:40,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T00:15:40,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:40,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:40,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:40,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:40,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:40,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:40,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T00:15:40,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:40,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975400984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:40,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:40,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975400994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:40,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:40,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:41,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/caca06085414442d9ccacaf5fb3c15e6 2024-11-19T00:15:41,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/a0c7714b738f41bf8ec1f764ee632a76 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/a0c7714b738f41bf8ec1f764ee632a76 2024-11-19T00:15:41,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/a0c7714b738f41bf8ec1f764ee632a76, entries=200, sequenceid=395, filesize=14.4 K 2024-11-19T00:15:41,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/fecaf7220f99456ab19f660118d4c981 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/fecaf7220f99456ab19f660118d4c981 2024-11-19T00:15:41,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/fecaf7220f99456ab19f660118d4c981, entries=150, sequenceid=395, filesize=12.0 K 2024-11-19T00:15:41,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/caca06085414442d9ccacaf5fb3c15e6 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/caca06085414442d9ccacaf5fb3c15e6 2024-11-19T00:15:41,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/caca06085414442d9ccacaf5fb3c15e6, entries=150, sequenceid=395, filesize=12.0 K 2024-11-19T00:15:41,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 874070de567718d211743b79a5f68965 in 458ms, sequenceid=395, compaction requested=true 2024-11-19T00:15:41,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:41,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:15:41,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:41,078 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:41,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:15:41,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:41,078 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:41,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:15:41,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:41,080 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:41,080 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:41,080 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/B is initiating minor compaction (all files) 2024-11-19T00:15:41,080 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/A is initiating minor compaction (all files) 2024-11-19T00:15:41,080 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/B in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:41,080 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/A in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:41,080 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/e0085f70b651473d927abf72f99822f4, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/d001084ff2df4256afeb16ea15c858be, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/fecaf7220f99456ab19f660118d4c981] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=36.8 K 2024-11-19T00:15:41,080 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/5da83c83bcfb4853bee47f275d9cd3f8, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/ccd2b84bcc7a4a7fa2e6a9f8761b322f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/a0c7714b738f41bf8ec1f764ee632a76] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=41.5 K 2024-11-19T00:15:41,080 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5da83c83bcfb4853bee47f275d9cd3f8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731975337814 2024-11-19T00:15:41,080 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting e0085f70b651473d927abf72f99822f4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731975337814 2024-11-19T00:15:41,080 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting d001084ff2df4256afeb16ea15c858be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731975338481 2024-11-19T00:15:41,080 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccd2b84bcc7a4a7fa2e6a9f8761b322f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731975338481 2024-11-19T00:15:41,081 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting fecaf7220f99456ab19f660118d4c981, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731975339843 2024-11-19T00:15:41,081 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0c7714b738f41bf8ec1f764ee632a76, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731975339843 2024-11-19T00:15:41,087 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#B#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:41,087 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#A#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:41,088 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/aced3b9aefe1442c93d1f8d9b309c5b9 is 50, key is test_row_0/B:col10/1731975339975/Put/seqid=0 2024-11-19T00:15:41,088 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/b56b5c2822374565b0b206eed40a6772 is 50, key is test_row_0/A:col10/1731975339975/Put/seqid=0 2024-11-19T00:15:41,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742339_1515 (size=13153) 2024-11-19T00:15:41,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742340_1516 (size=13153) 2024-11-19T00:15:41,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T00:15:41,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:41,118 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 874070de567718d211743b79a5f68965 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-19T00:15:41,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=A 2024-11-19T00:15:41,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:41,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=B 2024-11-19T00:15:41,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:41,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=C 2024-11-19T00:15:41,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:41,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/f68b26f04db749d2b525e6190afd4a94 is 50, key is test_row_1/A:col10/1731975340641/Put/seqid=0 2024-11-19T00:15:41,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742341_1517 (size=9857) 2024-11-19T00:15:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 874070de567718d211743b79a5f68965 2024-11-19T00:15:41,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:41,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975341347, max timestamp=1731975341369Num uniques 859; 2024-11-19T00:15:41,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975341347, max timestamp=1731975341369Num uniques 859; 2024-11-19T00:15:41,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:15:41,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:15:41,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975341347, max timestamp=1731975341369Num uniques 859; 2024-11-19T00:15:41,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:15:41,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975401442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:41,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:41,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975401446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:41,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:41,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:41,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:41,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T00:15:41,507 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/aced3b9aefe1442c93d1f8d9b309c5b9 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/aced3b9aefe1442c93d1f8d9b309c5b9 2024-11-19T00:15:41,510 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 874070de567718d211743b79a5f68965/B of 874070de567718d211743b79a5f68965 into aced3b9aefe1442c93d1f8d9b309c5b9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:41,510 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:41,511 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/B, priority=13, startTime=1731975341078; duration=0sec 2024-11-19T00:15:41,511 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:41,511 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:B 2024-11-19T00:15:41,511 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:41,511 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/b56b5c2822374565b0b206eed40a6772 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/b56b5c2822374565b0b206eed40a6772 2024-11-19T00:15:41,512 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:41,512 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/C is initiating minor compaction (all files) 2024-11-19T00:15:41,512 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/C in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:41,512 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/6cb229f8c53449d5a0b85e931fbd1646, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/7eb80b2cceb4486893cf5f4cb896bf58, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/caca06085414442d9ccacaf5fb3c15e6] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=36.8 K 2024-11-19T00:15:41,512 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cb229f8c53449d5a0b85e931fbd1646, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731975337814 2024-11-19T00:15:41,512 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 7eb80b2cceb4486893cf5f4cb896bf58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731975338481 2024-11-19T00:15:41,512 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting caca06085414442d9ccacaf5fb3c15e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731975339843 2024-11-19T00:15:41,515 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 874070de567718d211743b79a5f68965/A of 874070de567718d211743b79a5f68965 into b56b5c2822374565b0b206eed40a6772(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:41,515 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:41,515 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/A, priority=13, startTime=1731975341078; duration=0sec 2024-11-19T00:15:41,515 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:41,515 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:A 2024-11-19T00:15:41,520 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#C#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:41,521 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/4cdf810fb6e242099a762e3e7fff0b42 is 50, key is test_row_0/C:col10/1731975339975/Put/seqid=0 2024-11-19T00:15:41,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742342_1518 (size=13153) 2024-11-19T00:15:41,526 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/f68b26f04db749d2b525e6190afd4a94 2024-11-19T00:15:41,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/ec3f2c5f77b7461dbd685ed4f987e3b2 is 50, key is test_row_1/B:col10/1731975340641/Put/seqid=0 2024-11-19T00:15:41,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742343_1519 (size=9857) 2024-11-19T00:15:41,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975401553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:41,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:41,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:41,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:41,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:41,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975401563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:41,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:41,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:41,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:41,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:41,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975401773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:41,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:41,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:41,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:41,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975401783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:41,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:41,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:41,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:41,928 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/4cdf810fb6e242099a762e3e7fff0b42 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/4cdf810fb6e242099a762e3e7fff0b42 2024-11-19T00:15:41,932 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 874070de567718d211743b79a5f68965/C of 874070de567718d211743b79a5f68965 into 4cdf810fb6e242099a762e3e7fff0b42(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:41,932 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:41,933 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/C, priority=13, startTime=1731975341078; duration=0sec 2024-11-19T00:15:41,933 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:41,933 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:C 2024-11-19T00:15:41,934 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/ec3f2c5f77b7461dbd685ed4f987e3b2 2024-11-19T00:15:41,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/b2f1670169dd4df6b602bf25b2bfe218 is 50, key is test_row_1/C:col10/1731975340641/Put/seqid=0 2024-11-19T00:15:41,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742344_1520 (size=9857) 2024-11-19T00:15:41,947 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/b2f1670169dd4df6b602bf25b2bfe218 2024-11-19T00:15:41,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/f68b26f04db749d2b525e6190afd4a94 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/f68b26f04db749d2b525e6190afd4a94 2024-11-19T00:15:41,953 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/f68b26f04db749d2b525e6190afd4a94, entries=100, sequenceid=404, filesize=9.6 K 2024-11-19T00:15:41,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/ec3f2c5f77b7461dbd685ed4f987e3b2 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/ec3f2c5f77b7461dbd685ed4f987e3b2 2024-11-19T00:15:41,956 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/ec3f2c5f77b7461dbd685ed4f987e3b2, entries=100, sequenceid=404, filesize=9.6 K 2024-11-19T00:15:41,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/b2f1670169dd4df6b602bf25b2bfe218 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/b2f1670169dd4df6b602bf25b2bfe218 2024-11-19T00:15:41,959 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/b2f1670169dd4df6b602bf25b2bfe218, entries=100, sequenceid=404, filesize=9.6 K 2024-11-19T00:15:41,960 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 874070de567718d211743b79a5f68965 in 842ms, sequenceid=404, compaction requested=false 2024-11-19T00:15:41,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:41,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:41,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-19T00:15:41,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-19T00:15:41,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-19T00:15:41,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6130 sec 2024-11-19T00:15:41,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.6210 sec 2024-11-19T00:15:42,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 874070de567718d211743b79a5f68965 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-19T00:15:42,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=A 2024-11-19T00:15:42,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:15:42,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=B 2024-11-19T00:15:42,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:15:42,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=C 2024-11-19T00:15:42,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 874070de567718d211743b79a5f68965 2024-11-19T00:15:42,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:15:42,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/dc73eca1b26942ec8cffa56ea5cca288 is 50, key is test_row_0/A:col10/1731975341423/Put/seqid=0 2024-11-19T00:15:42,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742345_1521 (size=14741) 2024-11-19T00:15:42,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:42,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975402114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:42,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:42,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:42,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:42,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975402122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:42,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:42,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:42,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:42,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:42,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975402224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:42,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:42,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975402233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:42,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:42,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:42,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:42,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975402444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:42,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:42,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:42,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:42,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975402444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:42,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:42,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:42,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T00:15:42,494 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:15:42,494 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:15:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-19T00:15:42,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T00:15:42,496 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:15:42,496 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:15:42,496 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:15:42,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/dc73eca1b26942ec8cffa56ea5cca288 2024-11-19T00:15:42,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/20721a79c2b3429da0abd5714f207cb5 is 50, key is test_row_0/B:col10/1731975341423/Put/seqid=0 2024-11-19T00:15:42,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742346_1522 (size=12301) 2024-11-19T00:15:42,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T00:15:42,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T00:15:42,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:42,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:42,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:42,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:42,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:42,652 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T00:15:42,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975402763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:42,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975402763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:42,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T00:15:42,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:42,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:42,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:42,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:42,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:42,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:42,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T00:15:42,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/20721a79c2b3429da0abd5714f207cb5 2024-11-19T00:15:42,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/d29ba4153bac434d8e788fb2d02aecbb is 50, key is test_row_0/C:col10/1731975341423/Put/seqid=0 2024-11-19T00:15:42,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742347_1523 (size=12301) 2024-11-19T00:15:42,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T00:15:42,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:42,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:42,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:42,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:42,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:43,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T00:15:43,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:43,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:43,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:43,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:43,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:43,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:43,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T00:15:43,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T00:15:43,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:43,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:43,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:43,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:43,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:43,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:43,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60366 deadline: 1731975403284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:43,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:43,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:43,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:43,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:43,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1731975403284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:43,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 , the old value is region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:43,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:43,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., hostname=6b1531871d35,38567,1731975194317, seqNum=2 because the exception is null or not the one we care about 2024-11-19T00:15:43,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/d29ba4153bac434d8e788fb2d02aecbb 2024-11-19T00:15:43,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/dc73eca1b26942ec8cffa56ea5cca288 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/dc73eca1b26942ec8cffa56ea5cca288 2024-11-19T00:15:43,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/dc73eca1b26942ec8cffa56ea5cca288, entries=200, sequenceid=445, filesize=14.4 K 2024-11-19T00:15:43,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/20721a79c2b3429da0abd5714f207cb5 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/20721a79c2b3429da0abd5714f207cb5 2024-11-19T00:15:43,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/20721a79c2b3429da0abd5714f207cb5, entries=150, sequenceid=445, filesize=12.0 K 2024-11-19T00:15:43,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/d29ba4153bac434d8e788fb2d02aecbb as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/d29ba4153bac434d8e788fb2d02aecbb 2024-11-19T00:15:43,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/d29ba4153bac434d8e788fb2d02aecbb, entries=150, sequenceid=445, filesize=12.0 K 2024-11-19T00:15:43,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 874070de567718d211743b79a5f68965 in 1243ms, sequenceid=445, compaction requested=true 2024-11-19T00:15:43,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:43,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:15:43,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:43,332 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:43,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:15:43,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:43,332 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:43,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:15:43,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:43,333 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37751 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:43,333 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:43,333 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/B is initiating minor compaction (all files) 2024-11-19T00:15:43,333 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/A is initiating minor compaction (all files) 2024-11-19T00:15:43,333 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/A in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:43,333 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/B in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:43,333 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/aced3b9aefe1442c93d1f8d9b309c5b9, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/ec3f2c5f77b7461dbd685ed4f987e3b2, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/20721a79c2b3429da0abd5714f207cb5] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=34.5 K 2024-11-19T00:15:43,333 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/b56b5c2822374565b0b206eed40a6772, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/f68b26f04db749d2b525e6190afd4a94, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/dc73eca1b26942ec8cffa56ea5cca288] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=36.9 K 2024-11-19T00:15:43,333 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting aced3b9aefe1442c93d1f8d9b309c5b9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731975339843 2024-11-19T00:15:43,334 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting b56b5c2822374565b0b206eed40a6772, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731975339843 2024-11-19T00:15:43,334 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting ec3f2c5f77b7461dbd685ed4f987e3b2, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731975340641 2024-11-19T00:15:43,334 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting f68b26f04db749d2b525e6190afd4a94, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731975340641 2024-11-19T00:15:43,334 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 20721a79c2b3429da0abd5714f207cb5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731975341423 2024-11-19T00:15:43,334 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc73eca1b26942ec8cffa56ea5cca288, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731975341423 2024-11-19T00:15:43,341 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#A#compaction#447 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:43,341 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/ffaa71b039ca4423a5d90b6bc39058da is 50, key is test_row_0/A:col10/1731975341423/Put/seqid=0 2024-11-19T00:15:43,343 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#B#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:43,344 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/13781958cd6441bb880dd3b2cee2f6fa is 50, key is test_row_0/B:col10/1731975341423/Put/seqid=0 2024-11-19T00:15:43,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742348_1524 (size=13255) 2024-11-19T00:15:43,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742349_1525 (size=13255) 2024-11-19T00:15:43,358 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/13781958cd6441bb880dd3b2cee2f6fa as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/13781958cd6441bb880dd3b2cee2f6fa 2024-11-19T00:15:43,363 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 874070de567718d211743b79a5f68965/B of 874070de567718d211743b79a5f68965 into 13781958cd6441bb880dd3b2cee2f6fa(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:43,363 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:43,363 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/B, priority=13, startTime=1731975343332; duration=0sec 2024-11-19T00:15:43,363 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:43,363 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:B 2024-11-19T00:15:43,363 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:43,364 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:43,364 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/C is initiating minor compaction (all files) 2024-11-19T00:15:43,364 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/C in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:43,364 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/4cdf810fb6e242099a762e3e7fff0b42, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/b2f1670169dd4df6b602bf25b2bfe218, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/d29ba4153bac434d8e788fb2d02aecbb] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=34.5 K 2024-11-19T00:15:43,364 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cdf810fb6e242099a762e3e7fff0b42, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731975339843 2024-11-19T00:15:43,365 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting b2f1670169dd4df6b602bf25b2bfe218, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731975340641 2024-11-19T00:15:43,365 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting d29ba4153bac434d8e788fb2d02aecbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731975341423 2024-11-19T00:15:43,371 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#C#compaction#449 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:43,372 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/1206e0cca8e444ae88058986d2beda23 is 50, key is test_row_0/C:col10/1731975341423/Put/seqid=0 2024-11-19T00:15:43,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742350_1526 (size=13255) 2024-11-19T00:15:43,384 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/1206e0cca8e444ae88058986d2beda23 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/1206e0cca8e444ae88058986d2beda23 2024-11-19T00:15:43,390 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 874070de567718d211743b79a5f68965/C of 874070de567718d211743b79a5f68965 into 1206e0cca8e444ae88058986d2beda23(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:43,390 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:43,390 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/C, priority=13, startTime=1731975343332; duration=0sec 2024-11-19T00:15:43,390 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:43,390 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:C 2024-11-19T00:15:43,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T00:15:43,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:43,412 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 874070de567718d211743b79a5f68965 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T00:15:43,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=A 2024-11-19T00:15:43,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:43,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=B 2024-11-19T00:15:43,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:43,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=C 2024-11-19T00:15:43,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:43,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/48f18d4b10c94d229543ffe26f9681cf is 50, key is test_row_0/A:col10/1731975342100/Put/seqid=0 2024-11-19T00:15:43,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742351_1527 (size=12301) 2024-11-19T00:15:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T00:15:43,751 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/ffaa71b039ca4423a5d90b6bc39058da as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/ffaa71b039ca4423a5d90b6bc39058da 2024-11-19T00:15:43,758 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 874070de567718d211743b79a5f68965/A of 874070de567718d211743b79a5f68965 into ffaa71b039ca4423a5d90b6bc39058da(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:43,758 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:43,758 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/A, priority=13, startTime=1731975343332; duration=0sec 2024-11-19T00:15:43,758 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:43,758 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:A 2024-11-19T00:15:43,822 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/48f18d4b10c94d229543ffe26f9681cf 2024-11-19T00:15:43,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/ab5f58433c8a417bb013faabc5196ee9 is 50, key is test_row_0/B:col10/1731975342100/Put/seqid=0 2024-11-19T00:15:43,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742352_1528 (size=12301) 2024-11-19T00:15:44,231 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/ab5f58433c8a417bb013faabc5196ee9 2024-11-19T00:15:44,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/6946b2a693e34381b1843319b6f5c9de is 50, key is test_row_0/C:col10/1731975342100/Put/seqid=0 2024-11-19T00:15:44,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742353_1529 (size=12301) 2024-11-19T00:15:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 874070de567718d211743b79a5f68965 2024-11-19T00:15:44,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. as already flushing 2024-11-19T00:15:44,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975344342, max timestamp=1731975344365Num uniques 859; 2024-11-19T00:15:44,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975344342, max timestamp=1731975344365Num uniques 859; 2024-11-19T00:15:44,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975344342, max timestamp=1731975344365Num uniques 859; 2024-11-19T00:15:44,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:15:44,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:15:44,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:15:44,439 INFO [Thread-1975 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1975. 2024-11-19T00:15:44,439 DEBUG [Thread-1975 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:15:44,439 DEBUG [Thread-1975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:44,439 DEBUG [Thread-1975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:44,439 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:15:44,440 INFO [Thread-1977 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1977. 2024-11-19T00:15:44,440 DEBUG [Thread-1977 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:15:44,440 DEBUG [Thread-1977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:44,440 DEBUG [Thread-1977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:44,440 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:15:44,440 INFO [Thread-1986 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1986. 2024-11-19T00:15:44,440 DEBUG [Thread-1986 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:15:44,440 DEBUG [Thread-1986 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:44,440 DEBUG [Thread-1986 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:44,440 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:15:44,450 INFO [Thread-1992 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1992. 2024-11-19T00:15:44,450 DEBUG [Thread-1992 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:15:44,450 DEBUG [Thread-1992 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:44,450 DEBUG [Thread-1992 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:44,450 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:15:44,455 INFO [Thread-1990 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1990. 2024-11-19T00:15:44,455 DEBUG [Thread-1990 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:15:44,455 DEBUG [Thread-1990 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:44,455 DEBUG [Thread-1990 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:44,455 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:15:44,456 INFO [Thread-1988 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1988. 2024-11-19T00:15:44,456 DEBUG [Thread-1988 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:15:44,456 DEBUG [Thread-1988 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:44,456 DEBUG [Thread-1988 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:44,456 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:15:44,460 INFO [Thread-1994 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1994. 2024-11-19T00:15:44,460 DEBUG [Thread-1994 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:15:44,460 DEBUG [Thread-1994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:44,461 DEBUG [Thread-1994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:44,461 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:15:44,640 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/6946b2a693e34381b1843319b6f5c9de 2024-11-19T00:15:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T00:15:44,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/48f18d4b10c94d229543ffe26f9681cf as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/48f18d4b10c94d229543ffe26f9681cf 2024-11-19T00:15:44,651 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/48f18d4b10c94d229543ffe26f9681cf, entries=150, sequenceid=455, filesize=12.0 K 2024-11-19T00:15:44,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/ab5f58433c8a417bb013faabc5196ee9 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/ab5f58433c8a417bb013faabc5196ee9 2024-11-19T00:15:44,654 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/ab5f58433c8a417bb013faabc5196ee9, entries=150, sequenceid=455, filesize=12.0 K 2024-11-19T00:15:44,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/6946b2a693e34381b1843319b6f5c9de as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/6946b2a693e34381b1843319b6f5c9de 2024-11-19T00:15:44,658 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/6946b2a693e34381b1843319b6f5c9de, entries=150, sequenceid=455, filesize=12.0 K 2024-11-19T00:15:44,659 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=114.05 KB/116790 for 874070de567718d211743b79a5f68965 in 1247ms, sequenceid=455, compaction requested=false 2024-11-19T00:15:44,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:44,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:44,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-19T00:15:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-19T00:15:44,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-19T00:15:44,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1640 sec 2024-11-19T00:15:44,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.1680 sec 2024-11-19T00:15:44,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on 874070de567718d211743b79a5f68965 2024-11-19T00:15:44,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 874070de567718d211743b79a5f68965 3/3 column families, dataSize=120.76 KB heapSize=345.14 KB 2024-11-19T00:15:44,886 INFO [Thread-1981 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1981. 2024-11-19T00:15:44,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=A 2024-11-19T00:15:44,886 DEBUG [Thread-1981 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:15:44,886 DEBUG [Thread-1981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:44,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:44,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=B 2024-11-19T00:15:44,887 DEBUG [Thread-1981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:44,887 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:15:44,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:44,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=C 2024-11-19T00:15:44,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:44,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/46bc74b7951b43c89837ba486e0c5f71 is 50, key is test_row_0/A:col10/1731975344406/Put/seqid=0 2024-11-19T00:15:44,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742354_1530 (size=12301) 2024-11-19T00:15:44,915 INFO [Thread-1979 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1979. 2024-11-19T00:15:44,915 DEBUG [Thread-1979 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:15:44,915 DEBUG [Thread-1979 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:44,915 DEBUG [Thread-1979 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:44,915 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:15:44,925 INFO [Thread-1983 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1983. 2024-11-19T00:15:44,925 DEBUG [Thread-1983 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:15:44,925 DEBUG [Thread-1983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:44,926 DEBUG [Thread-1983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:44,926 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:15:45,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/46bc74b7951b43c89837ba486e0c5f71 2024-11-19T00:15:45,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/92286b0b167a4db8878c38503a4c7a1d is 50, key is test_row_0/B:col10/1731975344406/Put/seqid=0 2024-11-19T00:15:45,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742355_1531 (size=12301) 2024-11-19T00:15:45,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/92286b0b167a4db8878c38503a4c7a1d 2024-11-19T00:15:45,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/e918b880d0b74976bbe088c178da79ae is 50, key is test_row_0/C:col10/1731975344406/Put/seqid=0 2024-11-19T00:15:45,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742356_1532 (size=12301) 2024-11-19T00:15:46,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/e918b880d0b74976bbe088c178da79ae 2024-11-19T00:15:46,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/46bc74b7951b43c89837ba486e0c5f71 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/46bc74b7951b43c89837ba486e0c5f71 2024-11-19T00:15:46,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/46bc74b7951b43c89837ba486e0c5f71, entries=150, sequenceid=491, filesize=12.0 K 2024-11-19T00:15:46,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/92286b0b167a4db8878c38503a4c7a1d as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/92286b0b167a4db8878c38503a4c7a1d 2024-11-19T00:15:46,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/92286b0b167a4db8878c38503a4c7a1d, entries=150, sequenceid=491, filesize=12.0 K 2024-11-19T00:15:46,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/e918b880d0b74976bbe088c178da79ae as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/e918b880d0b74976bbe088c178da79ae 2024-11-19T00:15:46,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/e918b880d0b74976bbe088c178da79ae, entries=150, sequenceid=491, filesize=12.0 K 2024-11-19T00:15:46,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~345.09 KB/353376, currentSize=13.42 KB/13740 for 874070de567718d211743b79a5f68965 in 1261ms, sequenceid=491, compaction requested=true 2024-11-19T00:15:46,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:46,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:15:46,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:46,148 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:46,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:15:46,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:46,148 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:46,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 874070de567718d211743b79a5f68965:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:15:46,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:46,149 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:46,149 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/A is initiating minor compaction (all files) 2024-11-19T00:15:46,149 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:46,149 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/A in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:46,149 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/B is initiating minor compaction (all files) 2024-11-19T00:15:46,149 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/B in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:46,149 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/ffaa71b039ca4423a5d90b6bc39058da, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/48f18d4b10c94d229543ffe26f9681cf, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/46bc74b7951b43c89837ba486e0c5f71] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=37.0 K 2024-11-19T00:15:46,149 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/13781958cd6441bb880dd3b2cee2f6fa, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/ab5f58433c8a417bb013faabc5196ee9, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/92286b0b167a4db8878c38503a4c7a1d] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=37.0 K 2024-11-19T00:15:46,150 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffaa71b039ca4423a5d90b6bc39058da, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731975341423 2024-11-19T00:15:46,150 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 13781958cd6441bb880dd3b2cee2f6fa, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731975341423 2024-11-19T00:15:46,150 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48f18d4b10c94d229543ffe26f9681cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731975342100 2024-11-19T00:15:46,150 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting ab5f58433c8a417bb013faabc5196ee9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731975342100 2024-11-19T00:15:46,151 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46bc74b7951b43c89837ba486e0c5f71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1731975344406 2024-11-19T00:15:46,151 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 92286b0b167a4db8878c38503a4c7a1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1731975344406 2024-11-19T00:15:46,161 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#A#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:46,161 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#B#compaction#457 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:46,161 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/1c8ba111b761436a8c2ef20145a9fc32 is 50, key is test_row_0/B:col10/1731975344406/Put/seqid=0 2024-11-19T00:15:46,161 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/ca85b12ce331439a836fda24ba1d406b is 50, key is test_row_0/A:col10/1731975344406/Put/seqid=0 2024-11-19T00:15:46,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742357_1533 (size=13357) 2024-11-19T00:15:46,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742358_1534 (size=13357) 2024-11-19T00:15:46,576 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/1c8ba111b761436a8c2ef20145a9fc32 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/1c8ba111b761436a8c2ef20145a9fc32 2024-11-19T00:15:46,576 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/ca85b12ce331439a836fda24ba1d406b as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/ca85b12ce331439a836fda24ba1d406b 2024-11-19T00:15:46,581 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 874070de567718d211743b79a5f68965/B of 874070de567718d211743b79a5f68965 into 1c8ba111b761436a8c2ef20145a9fc32(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:46,581 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 874070de567718d211743b79a5f68965/A of 874070de567718d211743b79a5f68965 into ca85b12ce331439a836fda24ba1d406b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:46,581 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:46,581 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:46,581 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/A, priority=13, startTime=1731975346148; duration=0sec 2024-11-19T00:15:46,581 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/B, priority=13, startTime=1731975346148; duration=0sec 2024-11-19T00:15:46,581 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:46,581 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:B 2024-11-19T00:15:46,581 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:46,581 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:A 2024-11-19T00:15:46,581 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:46,582 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:46,582 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): 874070de567718d211743b79a5f68965/C is initiating minor compaction (all files) 2024-11-19T00:15:46,582 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 874070de567718d211743b79a5f68965/C in TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:46,582 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/1206e0cca8e444ae88058986d2beda23, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/6946b2a693e34381b1843319b6f5c9de, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/e918b880d0b74976bbe088c178da79ae] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp, totalSize=37.0 K 2024-11-19T00:15:46,582 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 1206e0cca8e444ae88058986d2beda23, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731975341423 2024-11-19T00:15:46,583 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 6946b2a693e34381b1843319b6f5c9de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731975342100 2024-11-19T00:15:46,583 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting e918b880d0b74976bbe088c178da79ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1731975344406 2024-11-19T00:15:46,590 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 874070de567718d211743b79a5f68965#C#compaction#458 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:46,590 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/84cf671c0e9c4423bd40ed63196154cc is 50, key is test_row_0/C:col10/1731975344406/Put/seqid=0 2024-11-19T00:15:46,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742359_1535 (size=13357) 2024-11-19T00:15:46,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T00:15:46,654 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:15:46,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T00:15:46,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 127 2024-11-19T00:15:46,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 131 2024-11-19T00:15:46,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-19T00:15:46,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-19T00:15:46,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-19T00:15:46,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T00:15:46,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T00:15:46,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1138 2024-11-19T00:15:46,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3404 rows 2024-11-19T00:15:46,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1143 2024-11-19T00:15:46,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3429 rows 2024-11-19T00:15:46,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1150 2024-11-19T00:15:46,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3416 rows 2024-11-19T00:15:46,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1146 2024-11-19T00:15:46,655 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3431 rows 2024-11-19T00:15:46,655 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1143 2024-11-19T00:15:46,655 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3429 rows 2024-11-19T00:15:46,655 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T00:15:46,655 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T00:15:46,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:46,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:46,655 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:15:46,656 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T00:15:46,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T00:15:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T00:15:46,658 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975346658"}]},"ts":"1731975346658"} 2024-11-19T00:15:46,659 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T00:15:46,659 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T00:15:46,660 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T00:15:46,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=874070de567718d211743b79a5f68965, UNASSIGN}] 2024-11-19T00:15:46,661 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=874070de567718d211743b79a5f68965, UNASSIGN 2024-11-19T00:15:46,662 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=874070de567718d211743b79a5f68965, regionState=CLOSING, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:15:46,663 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=874070de567718d211743b79a5f68965, UNASSIGN because future has completed 2024-11-19T00:15:46,663 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T00:15:46,663 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure 874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317}] 2024-11-19T00:15:46,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T00:15:46,816 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(122): Close 874070de567718d211743b79a5f68965 2024-11-19T00:15:46,816 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T00:15:46,816 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1722): Closing 874070de567718d211743b79a5f68965, disabling compactions & flushes 2024-11-19T00:15:46,817 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1993): waiting for 1 compactions to complete for region TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:46,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T00:15:47,006 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/84cf671c0e9c4423bd40ed63196154cc as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/84cf671c0e9c4423bd40ed63196154cc 2024-11-19T00:15:47,011 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 874070de567718d211743b79a5f68965/C of 874070de567718d211743b79a5f68965 into 84cf671c0e9c4423bd40ed63196154cc(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:47,011 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 874070de567718d211743b79a5f68965: 2024-11-19T00:15:47,011 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., storeName=874070de567718d211743b79a5f68965/C, priority=13, startTime=1731975346148; duration=0sec 2024-11-19T00:15:47,011 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:47,011 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:47,011 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:47,011 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. after waiting 0 ms 2024-11-19T00:15:47,011 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 874070de567718d211743b79a5f68965:C 2024-11-19T00:15:47,011 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:47,011 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(2902): Flushing 874070de567718d211743b79a5f68965 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-19T00:15:47,011 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=A 2024-11-19T00:15:47,011 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:47,011 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=B 2024-11-19T00:15:47,012 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:47,012 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 874070de567718d211743b79a5f68965, store=C 2024-11-19T00:15:47,012 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:47,015 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/0fb20203fdd945ce85afdcd90fb47fba is 50, key is test_row_0/A:col10/1731975344924/Put/seqid=0 2024-11-19T00:15:47,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742360_1536 (size=9857) 2024-11-19T00:15:47,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T00:15:47,421 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/0fb20203fdd945ce85afdcd90fb47fba 2024-11-19T00:15:47,433 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/42d193192f3a4adaa586d62ca093b44b is 50, key is test_row_0/B:col10/1731975344924/Put/seqid=0 2024-11-19T00:15:47,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742361_1537 (size=9857) 2024-11-19T00:15:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T00:15:47,838 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/42d193192f3a4adaa586d62ca093b44b 2024-11-19T00:15:47,851 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/42df2d56bbb64610a8c3789e30788ef8 is 50, key is test_row_0/C:col10/1731975344924/Put/seqid=0 2024-11-19T00:15:47,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742362_1538 (size=9857) 2024-11-19T00:15:48,256 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/42df2d56bbb64610a8c3789e30788ef8 2024-11-19T00:15:48,286 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/A/0fb20203fdd945ce85afdcd90fb47fba as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/0fb20203fdd945ce85afdcd90fb47fba 2024-11-19T00:15:48,292 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/0fb20203fdd945ce85afdcd90fb47fba, entries=100, sequenceid=499, filesize=9.6 K 2024-11-19T00:15:48,293 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/B/42d193192f3a4adaa586d62ca093b44b as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/42d193192f3a4adaa586d62ca093b44b 2024-11-19T00:15:48,297 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/42d193192f3a4adaa586d62ca093b44b, entries=100, sequenceid=499, filesize=9.6 K 2024-11-19T00:15:48,297 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/.tmp/C/42df2d56bbb64610a8c3789e30788ef8 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/42df2d56bbb64610a8c3789e30788ef8 2024-11-19T00:15:48,301 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/42df2d56bbb64610a8c3789e30788ef8, entries=100, sequenceid=499, filesize=9.6 K 2024-11-19T00:15:48,302 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 874070de567718d211743b79a5f68965 in 1291ms, sequenceid=499, compaction requested=false 2024-11-19T00:15:48,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/e9863581a5794de98492d2577bedf67a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/c125e4fd9717445c91ee02898e4addbf, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/6c8a7be84aea457a9f37cd8169c8759e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/00ee1a5126544ae0adeeb30397c2d6b8, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/edc800c4c67948da99ba75034b14494f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/5276bd7514de42faa3d537f4b82586c6, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/7da8d94c98f848019c93542bfa609bb3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/29c5b18ac6b14c62bdbd6533f3bcaa70, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/53a868b52b55465db530d086721e6eb3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/bcd5a2b81cf24a9896e1ecf8cd72a8da, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/920c84d4697f42a6937f131321e99872, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/6d2a6ebc91124117ae6db7ba5a6076b5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/d9bed3d42bb44967947221983c93ef11, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/b41090609ed24a94b7bd7756bd915d4a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/c89f5fd04b8744ebbe4335258e8529cb, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/e4e8763e970643a9a431ad0baf8e2d38, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/5863f9cdd856493780eac15e7a561a34, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/184b874dd86b46a2892e512dd34cdc81, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/ccb58ded1b2c458e9093dc20d583a2fb, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/08b5824dbccf4689ab22f9f40f96d2ab, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/cffd7e382d9d46afa91dba1e53e43181, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/970cf5fcf4eb4cb9b7cb3eeec482057b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/5da83c83bcfb4853bee47f275d9cd3f8, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/ccd2b84bcc7a4a7fa2e6a9f8761b322f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/a0c7714b738f41bf8ec1f764ee632a76, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/b56b5c2822374565b0b206eed40a6772, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/f68b26f04db749d2b525e6190afd4a94, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/dc73eca1b26942ec8cffa56ea5cca288, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/ffaa71b039ca4423a5d90b6bc39058da, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/48f18d4b10c94d229543ffe26f9681cf, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/46bc74b7951b43c89837ba486e0c5f71] to archive 2024-11-19T00:15:48,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T00:15:48,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/e9863581a5794de98492d2577bedf67a to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/e9863581a5794de98492d2577bedf67a 2024-11-19T00:15:48,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/c125e4fd9717445c91ee02898e4addbf to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/c125e4fd9717445c91ee02898e4addbf 2024-11-19T00:15:48,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/6c8a7be84aea457a9f37cd8169c8759e to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/6c8a7be84aea457a9f37cd8169c8759e 2024-11-19T00:15:48,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/00ee1a5126544ae0adeeb30397c2d6b8 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/00ee1a5126544ae0adeeb30397c2d6b8 2024-11-19T00:15:48,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/edc800c4c67948da99ba75034b14494f to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/edc800c4c67948da99ba75034b14494f 2024-11-19T00:15:48,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/5276bd7514de42faa3d537f4b82586c6 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/5276bd7514de42faa3d537f4b82586c6 2024-11-19T00:15:48,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/7da8d94c98f848019c93542bfa609bb3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/7da8d94c98f848019c93542bfa609bb3 2024-11-19T00:15:48,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/29c5b18ac6b14c62bdbd6533f3bcaa70 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/29c5b18ac6b14c62bdbd6533f3bcaa70 2024-11-19T00:15:48,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/53a868b52b55465db530d086721e6eb3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/53a868b52b55465db530d086721e6eb3 2024-11-19T00:15:48,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/bcd5a2b81cf24a9896e1ecf8cd72a8da to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/bcd5a2b81cf24a9896e1ecf8cd72a8da 2024-11-19T00:15:48,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/920c84d4697f42a6937f131321e99872 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/920c84d4697f42a6937f131321e99872 2024-11-19T00:15:48,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/6d2a6ebc91124117ae6db7ba5a6076b5 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/6d2a6ebc91124117ae6db7ba5a6076b5 2024-11-19T00:15:48,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/d9bed3d42bb44967947221983c93ef11 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/d9bed3d42bb44967947221983c93ef11 2024-11-19T00:15:48,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/b41090609ed24a94b7bd7756bd915d4a to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/b41090609ed24a94b7bd7756bd915d4a 2024-11-19T00:15:48,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/c89f5fd04b8744ebbe4335258e8529cb to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/c89f5fd04b8744ebbe4335258e8529cb 2024-11-19T00:15:48,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/e4e8763e970643a9a431ad0baf8e2d38 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/e4e8763e970643a9a431ad0baf8e2d38 2024-11-19T00:15:48,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/5863f9cdd856493780eac15e7a561a34 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/5863f9cdd856493780eac15e7a561a34 2024-11-19T00:15:48,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/184b874dd86b46a2892e512dd34cdc81 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/184b874dd86b46a2892e512dd34cdc81 2024-11-19T00:15:48,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/ccb58ded1b2c458e9093dc20d583a2fb to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/ccb58ded1b2c458e9093dc20d583a2fb 2024-11-19T00:15:48,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/08b5824dbccf4689ab22f9f40f96d2ab to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/08b5824dbccf4689ab22f9f40f96d2ab 2024-11-19T00:15:48,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/cffd7e382d9d46afa91dba1e53e43181 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/cffd7e382d9d46afa91dba1e53e43181 2024-11-19T00:15:48,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/970cf5fcf4eb4cb9b7cb3eeec482057b to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/970cf5fcf4eb4cb9b7cb3eeec482057b 2024-11-19T00:15:48,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/5da83c83bcfb4853bee47f275d9cd3f8 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/5da83c83bcfb4853bee47f275d9cd3f8 2024-11-19T00:15:48,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/ccd2b84bcc7a4a7fa2e6a9f8761b322f to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/ccd2b84bcc7a4a7fa2e6a9f8761b322f 2024-11-19T00:15:48,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/a0c7714b738f41bf8ec1f764ee632a76 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/a0c7714b738f41bf8ec1f764ee632a76 2024-11-19T00:15:48,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/b56b5c2822374565b0b206eed40a6772 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/b56b5c2822374565b0b206eed40a6772 2024-11-19T00:15:48,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/f68b26f04db749d2b525e6190afd4a94 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/f68b26f04db749d2b525e6190afd4a94 2024-11-19T00:15:48,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/dc73eca1b26942ec8cffa56ea5cca288 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/dc73eca1b26942ec8cffa56ea5cca288 2024-11-19T00:15:48,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/ffaa71b039ca4423a5d90b6bc39058da to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/ffaa71b039ca4423a5d90b6bc39058da 2024-11-19T00:15:48,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/48f18d4b10c94d229543ffe26f9681cf to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/48f18d4b10c94d229543ffe26f9681cf 2024-11-19T00:15:48,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/46bc74b7951b43c89837ba486e0c5f71 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/46bc74b7951b43c89837ba486e0c5f71 2024-11-19T00:15:48,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/b9f460760a8c4745b916a7ecdcbbe4d9, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/eff889601947415fb5a6b41dd18600c2, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/36d72c9860e44818a861a3922e6db24f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/c578fc6d454b4cc5bd3c7d1905dd5b36, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/9d42fb75fca74ac89809a21efd9476b6, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/140a1e857bdf4b3495cfb72a25dd8436, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/65162ccd36264b29a1271e12790e59ed, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/ef7f80599a8d482e859083ecdaf84fcc, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/5d9b1e6ff0b0479085f05556955aa68e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/e568384efd5046ed9f7a2b367d40bd05, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/2c3cc9539e5e4f5b8a4d62c8e547f321, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/1066c5c8a3ce44208d39b90619abc85c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/4e5ee7c103fe4e8b909d7eadf6114c36, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/948107b880914ee7acacc9039a6c24bf, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/330c9ada012f4a8e9ccc3e4740237142, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/7da7820d8b324d3a8dea4e29cacb7813, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/216e940344fc440e975c7ad7078810e1, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/80c6ac8d30c64b15bde6b1af139390ad, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/5ee5e82355364aacb02bdfad0cfc0318, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/1d14d4a0192643c891add1de5b68db5d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/78616000ac7345889894f8c1d3ecba67, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/e0085f70b651473d927abf72f99822f4, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/11cafdb53bd34a2b9ad2fd33fb546d25, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/d001084ff2df4256afeb16ea15c858be, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/aced3b9aefe1442c93d1f8d9b309c5b9, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/fecaf7220f99456ab19f660118d4c981, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/ec3f2c5f77b7461dbd685ed4f987e3b2, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/13781958cd6441bb880dd3b2cee2f6fa, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/20721a79c2b3429da0abd5714f207cb5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/ab5f58433c8a417bb013faabc5196ee9, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/92286b0b167a4db8878c38503a4c7a1d] to archive 2024-11-19T00:15:48,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T00:15:48,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/b9f460760a8c4745b916a7ecdcbbe4d9 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/b9f460760a8c4745b916a7ecdcbbe4d9 2024-11-19T00:15:48,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/eff889601947415fb5a6b41dd18600c2 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/eff889601947415fb5a6b41dd18600c2 2024-11-19T00:15:48,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/36d72c9860e44818a861a3922e6db24f to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/36d72c9860e44818a861a3922e6db24f 2024-11-19T00:15:48,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/c578fc6d454b4cc5bd3c7d1905dd5b36 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/c578fc6d454b4cc5bd3c7d1905dd5b36 2024-11-19T00:15:48,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/9d42fb75fca74ac89809a21efd9476b6 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/9d42fb75fca74ac89809a21efd9476b6 2024-11-19T00:15:48,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/140a1e857bdf4b3495cfb72a25dd8436 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/140a1e857bdf4b3495cfb72a25dd8436 2024-11-19T00:15:48,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/65162ccd36264b29a1271e12790e59ed to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/65162ccd36264b29a1271e12790e59ed 2024-11-19T00:15:48,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/ef7f80599a8d482e859083ecdaf84fcc to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/ef7f80599a8d482e859083ecdaf84fcc 2024-11-19T00:15:48,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/5d9b1e6ff0b0479085f05556955aa68e to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/5d9b1e6ff0b0479085f05556955aa68e 2024-11-19T00:15:48,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/e568384efd5046ed9f7a2b367d40bd05 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/e568384efd5046ed9f7a2b367d40bd05 2024-11-19T00:15:48,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/2c3cc9539e5e4f5b8a4d62c8e547f321 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/2c3cc9539e5e4f5b8a4d62c8e547f321 2024-11-19T00:15:48,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/1066c5c8a3ce44208d39b90619abc85c to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/1066c5c8a3ce44208d39b90619abc85c 2024-11-19T00:15:48,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/4e5ee7c103fe4e8b909d7eadf6114c36 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/4e5ee7c103fe4e8b909d7eadf6114c36 2024-11-19T00:15:48,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/948107b880914ee7acacc9039a6c24bf to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/948107b880914ee7acacc9039a6c24bf 2024-11-19T00:15:48,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/330c9ada012f4a8e9ccc3e4740237142 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/330c9ada012f4a8e9ccc3e4740237142 2024-11-19T00:15:48,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/7da7820d8b324d3a8dea4e29cacb7813 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/7da7820d8b324d3a8dea4e29cacb7813 2024-11-19T00:15:48,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/216e940344fc440e975c7ad7078810e1 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/216e940344fc440e975c7ad7078810e1 2024-11-19T00:15:48,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/80c6ac8d30c64b15bde6b1af139390ad to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/80c6ac8d30c64b15bde6b1af139390ad 2024-11-19T00:15:48,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/5ee5e82355364aacb02bdfad0cfc0318 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/5ee5e82355364aacb02bdfad0cfc0318 2024-11-19T00:15:48,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/1d14d4a0192643c891add1de5b68db5d to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/1d14d4a0192643c891add1de5b68db5d 2024-11-19T00:15:48,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/78616000ac7345889894f8c1d3ecba67 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/78616000ac7345889894f8c1d3ecba67 2024-11-19T00:15:48,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/e0085f70b651473d927abf72f99822f4 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/e0085f70b651473d927abf72f99822f4 2024-11-19T00:15:48,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/11cafdb53bd34a2b9ad2fd33fb546d25 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/11cafdb53bd34a2b9ad2fd33fb546d25 2024-11-19T00:15:48,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/d001084ff2df4256afeb16ea15c858be to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/d001084ff2df4256afeb16ea15c858be 2024-11-19T00:15:48,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/aced3b9aefe1442c93d1f8d9b309c5b9 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/aced3b9aefe1442c93d1f8d9b309c5b9 2024-11-19T00:15:48,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/fecaf7220f99456ab19f660118d4c981 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/fecaf7220f99456ab19f660118d4c981 2024-11-19T00:15:48,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/ec3f2c5f77b7461dbd685ed4f987e3b2 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/ec3f2c5f77b7461dbd685ed4f987e3b2 2024-11-19T00:15:48,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/13781958cd6441bb880dd3b2cee2f6fa to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/13781958cd6441bb880dd3b2cee2f6fa 2024-11-19T00:15:48,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/20721a79c2b3429da0abd5714f207cb5 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/20721a79c2b3429da0abd5714f207cb5 2024-11-19T00:15:48,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/ab5f58433c8a417bb013faabc5196ee9 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/ab5f58433c8a417bb013faabc5196ee9 2024-11-19T00:15:48,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/92286b0b167a4db8878c38503a4c7a1d to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/92286b0b167a4db8878c38503a4c7a1d 2024-11-19T00:15:48,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/6bb2ad7621934df1ad9ff4d5715ecc5f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/fa078044348a4460a6d90e0378082e5f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/710371943140466b84e2dab677cb2a43, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/fabcf6684e264d0eb968f0eb78b17af3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/f0d1299641fa4ec1a0fa6568130a57ce, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/56f37fe157a34d99945d703190f5b426, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/1f0e5660530c4b2daac5c371a7033720, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/e038e3a2e50d4598b8479a0e54550935, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/450fcd3f75894ab8908b4f7dde76dd50, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/fb6f43556d7a4bed989eca7e1c11a900, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/91514e9dd19a45abb8067cbabc4e4ca5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/986481afbb26483396b8c9a9b10398ae, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/e4922fccd11b4a9c96e9277ff04c4839, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/319705a7e3454d97a5823eac3ed8fd15, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/363673d1efd64b56a5b948fcc89aff56, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/82e2225764a646df9bac9717fa98b542, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/d0d415c53d384cfabdd817a190634154, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/95d5effa99c24d4fa45b685fa844a5f6, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/ec0dbef8debb4d5ebc06c8dc96c105b8, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/f91ff4b6c0524d999f6373ecee30b17e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/da93a31547da4f20aa28dc1ea515e9b3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/6cb229f8c53449d5a0b85e931fbd1646, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/8e9f2e7cc58a40e095322e604920dc06, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/7eb80b2cceb4486893cf5f4cb896bf58, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/4cdf810fb6e242099a762e3e7fff0b42, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/caca06085414442d9ccacaf5fb3c15e6, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/b2f1670169dd4df6b602bf25b2bfe218, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/1206e0cca8e444ae88058986d2beda23, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/d29ba4153bac434d8e788fb2d02aecbb, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/6946b2a693e34381b1843319b6f5c9de, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/e918b880d0b74976bbe088c178da79ae] to archive 2024-11-19T00:15:48,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T00:15:48,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/6bb2ad7621934df1ad9ff4d5715ecc5f to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/6bb2ad7621934df1ad9ff4d5715ecc5f 2024-11-19T00:15:48,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/fa078044348a4460a6d90e0378082e5f to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/fa078044348a4460a6d90e0378082e5f 2024-11-19T00:15:48,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/710371943140466b84e2dab677cb2a43 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/710371943140466b84e2dab677cb2a43 2024-11-19T00:15:48,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/fabcf6684e264d0eb968f0eb78b17af3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/fabcf6684e264d0eb968f0eb78b17af3 2024-11-19T00:15:48,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/f0d1299641fa4ec1a0fa6568130a57ce to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/f0d1299641fa4ec1a0fa6568130a57ce 2024-11-19T00:15:48,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/56f37fe157a34d99945d703190f5b426 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/56f37fe157a34d99945d703190f5b426 2024-11-19T00:15:48,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/1f0e5660530c4b2daac5c371a7033720 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/1f0e5660530c4b2daac5c371a7033720 2024-11-19T00:15:48,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/e038e3a2e50d4598b8479a0e54550935 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/e038e3a2e50d4598b8479a0e54550935 2024-11-19T00:15:48,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/450fcd3f75894ab8908b4f7dde76dd50 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/450fcd3f75894ab8908b4f7dde76dd50 2024-11-19T00:15:48,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/fb6f43556d7a4bed989eca7e1c11a900 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/fb6f43556d7a4bed989eca7e1c11a900 2024-11-19T00:15:48,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/91514e9dd19a45abb8067cbabc4e4ca5 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/91514e9dd19a45abb8067cbabc4e4ca5 2024-11-19T00:15:48,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/986481afbb26483396b8c9a9b10398ae to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/986481afbb26483396b8c9a9b10398ae 2024-11-19T00:15:48,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/e4922fccd11b4a9c96e9277ff04c4839 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/e4922fccd11b4a9c96e9277ff04c4839 2024-11-19T00:15:48,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/319705a7e3454d97a5823eac3ed8fd15 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/319705a7e3454d97a5823eac3ed8fd15 2024-11-19T00:15:48,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/363673d1efd64b56a5b948fcc89aff56 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/363673d1efd64b56a5b948fcc89aff56 2024-11-19T00:15:48,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/82e2225764a646df9bac9717fa98b542 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/82e2225764a646df9bac9717fa98b542 2024-11-19T00:15:48,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/d0d415c53d384cfabdd817a190634154 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/d0d415c53d384cfabdd817a190634154 2024-11-19T00:15:48,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/95d5effa99c24d4fa45b685fa844a5f6 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/95d5effa99c24d4fa45b685fa844a5f6 2024-11-19T00:15:48,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/ec0dbef8debb4d5ebc06c8dc96c105b8 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/ec0dbef8debb4d5ebc06c8dc96c105b8 2024-11-19T00:15:48,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/f91ff4b6c0524d999f6373ecee30b17e to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/f91ff4b6c0524d999f6373ecee30b17e 2024-11-19T00:15:48,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/da93a31547da4f20aa28dc1ea515e9b3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/da93a31547da4f20aa28dc1ea515e9b3 2024-11-19T00:15:48,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/6cb229f8c53449d5a0b85e931fbd1646 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/6cb229f8c53449d5a0b85e931fbd1646 2024-11-19T00:15:48,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/8e9f2e7cc58a40e095322e604920dc06 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/8e9f2e7cc58a40e095322e604920dc06 2024-11-19T00:15:48,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/7eb80b2cceb4486893cf5f4cb896bf58 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/7eb80b2cceb4486893cf5f4cb896bf58 2024-11-19T00:15:48,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/4cdf810fb6e242099a762e3e7fff0b42 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/4cdf810fb6e242099a762e3e7fff0b42 2024-11-19T00:15:48,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/caca06085414442d9ccacaf5fb3c15e6 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/caca06085414442d9ccacaf5fb3c15e6 2024-11-19T00:15:48,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/b2f1670169dd4df6b602bf25b2bfe218 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/b2f1670169dd4df6b602bf25b2bfe218 2024-11-19T00:15:48,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/1206e0cca8e444ae88058986d2beda23 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/1206e0cca8e444ae88058986d2beda23 2024-11-19T00:15:48,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/d29ba4153bac434d8e788fb2d02aecbb to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/d29ba4153bac434d8e788fb2d02aecbb 2024-11-19T00:15:48,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/6946b2a693e34381b1843319b6f5c9de to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/6946b2a693e34381b1843319b6f5c9de 2024-11-19T00:15:48,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/e918b880d0b74976bbe088c178da79ae to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/e918b880d0b74976bbe088c178da79ae 2024-11-19T00:15:48,454 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/recovered.edits/502.seqid, newMaxSeqId=502, maxSeqId=1 2024-11-19T00:15:48,455 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. 2024-11-19T00:15:48,455 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1676): Region close journal for 874070de567718d211743b79a5f68965: Waiting for close lock at 1731975346816Running coprocessor pre-close hooks at 1731975346816Disabling compacts and flushes for region at 1731975346816Disabling writes for close at 1731975347011 (+195 ms)Obtaining lock to block concurrent updates at 1731975347011Preparing flush snapshotting stores in 874070de567718d211743b79a5f68965 at 1731975347011Finished memstore snapshotting TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731975347012 (+1 ms)Flushing stores of TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965. at 1731975347012Flushing 874070de567718d211743b79a5f68965/A: creating writer at 1731975347012Flushing 874070de567718d211743b79a5f68965/A: appending metadata at 1731975347015 (+3 ms)Flushing 874070de567718d211743b79a5f68965/A: closing flushed file at 1731975347015Flushing 874070de567718d211743b79a5f68965/B: creating writer at 1731975347430 (+415 ms)Flushing 874070de567718d211743b79a5f68965/B: appending metadata at 1731975347432 (+2 ms)Flushing 874070de567718d211743b79a5f68965/B: closing flushed file at 1731975347433 (+1 ms)Flushing 874070de567718d211743b79a5f68965/C: creating writer at 1731975347846 (+413 ms)Flushing 874070de567718d211743b79a5f68965/C: appending metadata at 1731975347850 (+4 ms)Flushing 874070de567718d211743b79a5f68965/C: closing flushed file at 1731975347851 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ee41465: reopening flushed file at 1731975348285 (+434 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50530314: reopening flushed file at 1731975348292 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a180655: reopening flushed file at 1731975348297 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 874070de567718d211743b79a5f68965 in 1291ms, sequenceid=499, compaction requested=false at 1731975348302 (+5 ms)Writing region close event to WAL at 1731975348450 (+148 ms)Running coprocessor post-close hooks at 1731975348455 (+5 ms)Closed at 1731975348455 2024-11-19T00:15:48,458 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(157): Closed 874070de567718d211743b79a5f68965 2024-11-19T00:15:48,459 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=874070de567718d211743b79a5f68965, regionState=CLOSED 2024-11-19T00:15:48,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure 874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 because future has completed 2024-11-19T00:15:48,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-19T00:15:48,466 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseRegionProcedure 874070de567718d211743b79a5f68965, server=6b1531871d35,38567,1731975194317 in 1.8000 sec 2024-11-19T00:15:48,469 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-19T00:15:48,469 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=874070de567718d211743b79a5f68965, UNASSIGN in 1.8050 sec 2024-11-19T00:15:48,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-19T00:15:48,473 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8100 sec 2024-11-19T00:15:48,474 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975348474"}]},"ts":"1731975348474"} 2024-11-19T00:15:48,476 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T00:15:48,476 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T00:15:48,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8210 sec 2024-11-19T00:15:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T00:15:48,805 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:15:48,807 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T00:15:48,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:15:48,812 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:15:48,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T00:15:48,813 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=135, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:15:48,816 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965 2024-11-19T00:15:48,819 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A, FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B, FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C, FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/recovered.edits] 2024-11-19T00:15:48,821 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/0fb20203fdd945ce85afdcd90fb47fba to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/0fb20203fdd945ce85afdcd90fb47fba 2024-11-19T00:15:48,822 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/ca85b12ce331439a836fda24ba1d406b to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/A/ca85b12ce331439a836fda24ba1d406b 2024-11-19T00:15:48,823 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/1c8ba111b761436a8c2ef20145a9fc32 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/1c8ba111b761436a8c2ef20145a9fc32 2024-11-19T00:15:48,824 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/42d193192f3a4adaa586d62ca093b44b to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/B/42d193192f3a4adaa586d62ca093b44b 2024-11-19T00:15:48,826 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/42df2d56bbb64610a8c3789e30788ef8 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/42df2d56bbb64610a8c3789e30788ef8 2024-11-19T00:15:48,826 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/84cf671c0e9c4423bd40ed63196154cc to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/C/84cf671c0e9c4423bd40ed63196154cc 2024-11-19T00:15:48,828 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/recovered.edits/502.seqid to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965/recovered.edits/502.seqid 2024-11-19T00:15:48,828 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/874070de567718d211743b79a5f68965 2024-11-19T00:15:48,828 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T00:15:48,829 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=135, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:15:48,831 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T00:15:48,833 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T00:15:48,834 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=135, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:15:48,834 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T00:15:48,834 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731975348834"}]},"ts":"9223372036854775807"} 2024-11-19T00:15:48,837 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T00:15:48,837 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 874070de567718d211743b79a5f68965, NAME => 'TestAcidGuarantees,,1731975322191.874070de567718d211743b79a5f68965.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T00:15:48,837 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T00:15:48,837 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731975348837"}]},"ts":"9223372036854775807"} 2024-11-19T00:15:48,839 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T00:15:48,839 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=135, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:15:48,841 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 33 msec 2024-11-19T00:15:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T00:15:48,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T00:15:48,924 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:15:48,942 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252 (was 250) - Thread LEAK? -, OpenFileDescriptor=455 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=492 (was 568), ProcessCount=11 (was 11), AvailableMemoryMB=1342 (was 1507) 2024-11-19T00:15:48,952 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=252, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=492, ProcessCount=11, AvailableMemoryMB=1341 2024-11-19T00:15:48,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T00:15:48,954 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T00:15:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T00:15:48,956 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T00:15:48,956 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:48,956 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 136 2024-11-19T00:15:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-19T00:15:48,957 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T00:15:48,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742363_1539 (size=1061) 2024-11-19T00:15:49,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-19T00:15:49,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-19T00:15:49,365 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2 2024-11-19T00:15:49,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742364_1540 (size=53) 2024-11-19T00:15:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-19T00:15:49,769 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:15:49,769 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing cfb45e8932b673e843a8031625dfee57, disabling compactions & flushes 2024-11-19T00:15:49,770 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:49,770 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:49,770 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. after waiting 0 ms 2024-11-19T00:15:49,770 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:49,770 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:49,770 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for cfb45e8932b673e843a8031625dfee57: Waiting for close lock at 1731975349769Disabling compacts and flushes for region at 1731975349769Disabling writes for close at 1731975349770 (+1 ms)Writing region close event to WAL at 1731975349770Closed at 1731975349770 2024-11-19T00:15:49,771 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T00:15:49,771 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731975349771"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731975349771"}]},"ts":"1731975349771"} 2024-11-19T00:15:49,772 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T00:15:49,773 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T00:15:49,773 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975349773"}]},"ts":"1731975349773"} 2024-11-19T00:15:49,774 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T00:15:49,774 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfb45e8932b673e843a8031625dfee57, ASSIGN}] 2024-11-19T00:15:49,775 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfb45e8932b673e843a8031625dfee57, ASSIGN 2024-11-19T00:15:49,776 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfb45e8932b673e843a8031625dfee57, ASSIGN; state=OFFLINE, location=6b1531871d35,38567,1731975194317; forceNewPlan=false, retain=false 2024-11-19T00:15:49,927 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=cfb45e8932b673e843a8031625dfee57, regionState=OPENING, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:15:49,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfb45e8932b673e843a8031625dfee57, ASSIGN because future has completed 2024-11-19T00:15:49,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317}] 2024-11-19T00:15:50,086 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:50,086 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7752): Opening region: {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} 2024-11-19T00:15:50,087 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:50,087 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:15:50,087 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7794): checking encryption for cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:50,087 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7797): checking classloading for cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:50,090 INFO [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:50,092 INFO [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:15:50,093 INFO [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfb45e8932b673e843a8031625dfee57 columnFamilyName A 2024-11-19T00:15:50,093 DEBUG [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:50,094 INFO [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] regionserver.HStore(327): Store=cfb45e8932b673e843a8031625dfee57/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:15:50,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-19T00:15:50,094 INFO [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:50,096 INFO [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:15:50,096 INFO [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfb45e8932b673e843a8031625dfee57 columnFamilyName B 2024-11-19T00:15:50,096 DEBUG [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:50,097 INFO [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] regionserver.HStore(327): Store=cfb45e8932b673e843a8031625dfee57/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:15:50,097 INFO [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:50,098 INFO [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:15:50,098 INFO [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfb45e8932b673e843a8031625dfee57 columnFamilyName C 2024-11-19T00:15:50,098 DEBUG [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:50,099 INFO [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] regionserver.HStore(327): Store=cfb45e8932b673e843a8031625dfee57/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:15:50,099 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:50,099 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1038): replaying wal for cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:50,100 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:50,100 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:50,102 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1048): stopping wal replay for cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:50,102 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1060): Cleaning up temporary data for cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:50,103 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T00:15:50,105 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1093): writing seq id for cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:50,108 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T00:15:50,108 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1114): Opened cfb45e8932b673e843a8031625dfee57; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73152285, jitterRate=0.090053990483284}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T00:15:50,108 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:50,109 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1006): Region open journal for cfb45e8932b673e843a8031625dfee57: Running coprocessor pre-open hook at 1731975350088Writing region info on filesystem at 1731975350088Initializing all the Stores at 1731975350089 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975350089Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975350089Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975350090 (+1 ms)Cleaning up temporary data from old regions at 1731975350102 (+12 ms)Running coprocessor post-open hooks at 1731975350108 (+6 ms)Region opened successfully at 1731975350109 (+1 ms) 2024-11-19T00:15:50,110 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., pid=138, masterSystemTime=1731975350081 2024-11-19T00:15:50,112 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:50,112 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:50,113 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=cfb45e8932b673e843a8031625dfee57, regionState=OPEN, openSeqNum=2, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:15:50,114 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 because future has completed 2024-11-19T00:15:50,116 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-19T00:15:50,117 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 in 187 msec 2024-11-19T00:15:50,119 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-19T00:15:50,119 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfb45e8932b673e843a8031625dfee57, ASSIGN in 343 msec 2024-11-19T00:15:50,121 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T00:15:50,121 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975350121"}]},"ts":"1731975350121"} 2024-11-19T00:15:50,123 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T00:15:50,123 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T00:15:50,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1690 sec 2024-11-19T00:15:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-19T00:15:51,104 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:15:51,105 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e76e6cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,105 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:15:51,105 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:15:51,106 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:15:51,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:15:51,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:15:51,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@540c6102, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:15:51,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:15:51,108 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:51,109 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:15:51,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@511266fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,110 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:15:51,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:15:51,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:15:51,112 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:15:51,113 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T00:15:51,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 6b1531871d35,33843,1731975193570 2024-11-19T00:15:51,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@61ea2f9e 2024-11-19T00:15:51,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T00:15:51,115 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T00:15:51,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T00:15:51,117 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T00:15:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T00:15:51,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742365_1541 (size=1097) 2024-11-19T00:15:51,135 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-19T00:15:51,135 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-19T00:15:51,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T00:15:51,139 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfb45e8932b673e843a8031625dfee57, REOPEN/MOVE}] 2024-11-19T00:15:51,140 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfb45e8932b673e843a8031625dfee57, REOPEN/MOVE 2024-11-19T00:15:51,141 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=cfb45e8932b673e843a8031625dfee57, regionState=CLOSING, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:15:51,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfb45e8932b673e843a8031625dfee57, REOPEN/MOVE because future has completed 2024-11-19T00:15:51,142 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T00:15:51,142 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317}] 2024-11-19T00:15:51,295 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:51,295 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T00:15:51,295 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing cfb45e8932b673e843a8031625dfee57, disabling compactions & flushes 2024-11-19T00:15:51,295 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:51,295 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:51,295 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. after waiting 0 ms 2024-11-19T00:15:51,295 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:51,299 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T00:15:51,299 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:51,299 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for cfb45e8932b673e843a8031625dfee57: Waiting for close lock at 1731975351295Running coprocessor pre-close hooks at 1731975351295Disabling compacts and flushes for region at 1731975351295Disabling writes for close at 1731975351295Writing region close event to WAL at 1731975351296 (+1 ms)Running coprocessor post-close hooks at 1731975351299 (+3 ms)Closed at 1731975351299 2024-11-19T00:15:51,299 WARN [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionServer(3299): Not adding moved region record: cfb45e8932b673e843a8031625dfee57 to self. 2024-11-19T00:15:51,301 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:51,302 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=cfb45e8932b673e843a8031625dfee57, regionState=CLOSED 2024-11-19T00:15:51,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 because future has completed 2024-11-19T00:15:51,307 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-19T00:15:51,307 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 in 163 msec 2024-11-19T00:15:51,308 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfb45e8932b673e843a8031625dfee57, REOPEN/MOVE; state=CLOSED, location=6b1531871d35,38567,1731975194317; forceNewPlan=false, retain=true 2024-11-19T00:15:51,459 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=cfb45e8932b673e843a8031625dfee57, regionState=OPENING, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:15:51,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfb45e8932b673e843a8031625dfee57, REOPEN/MOVE because future has completed 2024-11-19T00:15:51,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317}] 2024-11-19T00:15:51,617 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:51,617 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7752): Opening region: {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} 2024-11-19T00:15:51,617 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:51,617 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T00:15:51,618 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7794): checking encryption for cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:51,618 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7797): checking classloading for cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:51,622 INFO [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:51,622 INFO [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:15:51,623 INFO [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfb45e8932b673e843a8031625dfee57 columnFamilyName A 2024-11-19T00:15:51,624 DEBUG [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:51,624 INFO [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] regionserver.HStore(327): Store=cfb45e8932b673e843a8031625dfee57/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:15:51,625 INFO [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:51,625 INFO [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:15:51,626 INFO [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfb45e8932b673e843a8031625dfee57 columnFamilyName B 2024-11-19T00:15:51,626 DEBUG [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:51,626 INFO [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] regionserver.HStore(327): Store=cfb45e8932b673e843a8031625dfee57/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:15:51,626 INFO [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:51,627 INFO [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T00:15:51,627 INFO [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfb45e8932b673e843a8031625dfee57 columnFamilyName C 2024-11-19T00:15:51,627 DEBUG [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:51,627 INFO [StoreOpener-cfb45e8932b673e843a8031625dfee57-1 {}] regionserver.HStore(327): Store=cfb45e8932b673e843a8031625dfee57/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T00:15:51,627 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:51,628 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1038): replaying wal for cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:51,628 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:51,629 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:51,630 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1048): stopping wal replay for cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:51,630 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1060): Cleaning up temporary data for cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:51,631 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T00:15:51,632 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1093): writing seq id for cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:51,633 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1114): Opened cfb45e8932b673e843a8031625dfee57; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70641376, jitterRate=0.05263853073120117}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T00:15:51,633 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:51,634 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1006): Region open journal for cfb45e8932b673e843a8031625dfee57: Running coprocessor pre-open hook at 1731975351618Writing region info on filesystem at 1731975351618Initializing all the Stores at 1731975351621 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975351621Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975351621Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731975351622 (+1 ms)Cleaning up temporary data from old regions at 1731975351630 (+8 ms)Running coprocessor post-open hooks at 1731975351633 (+3 ms)Region opened successfully at 1731975351634 (+1 ms) 2024-11-19T00:15:51,635 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., pid=143, masterSystemTime=1731975351614 2024-11-19T00:15:51,636 DEBUG [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:51,637 INFO [RS_OPEN_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:51,637 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=cfb45e8932b673e843a8031625dfee57, regionState=OPEN, openSeqNum=5, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:15:51,638 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 because future has completed 2024-11-19T00:15:51,641 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=141 2024-11-19T00:15:51,641 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 in 178 msec 2024-11-19T00:15:51,644 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-19T00:15:51,644 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfb45e8932b673e843a8031625dfee57, REOPEN/MOVE in 502 msec 2024-11-19T00:15:51,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-19T00:15:51,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 507 msec 2024-11-19T00:15:51,648 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 529 msec 2024-11-19T00:15:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T00:15:51,649 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:15:51,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,650 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:15:51,650 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:15:51,651 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:15:51,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:15:51,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:15:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ad7f0ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:15:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:15:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:51,653 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:15:51,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61f8999c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,655 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:15:51,655 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:15:51,656 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:15:51,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:15:51,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:15:51,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4483ac25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:15:51,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:15:51,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:51,658 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:15:51,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dc7d35a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:15:51,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:15:51,660 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:15:51,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:15:51,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:15:51,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c778afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:15:51,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:15:51,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:51,662 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:15:51,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45defeae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:15:51,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:15:51,667 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:15:51,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:15:51,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:15:51,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6979dc49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:15:51,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:15:51,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:51,668 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:15:51,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc3aaaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0b6ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:15:51,670 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:15:51,671 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:15:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:15:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:15:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40c07a76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:15:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:15:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:51,673 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:15:51,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cb5e7cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,674 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:15:51,674 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:15:51,675 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:15:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:15:51,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:15:51,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ce78aee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:15:51,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:15:51,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:51,677 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:15:51,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ed4e770, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:15:51,678 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:15:51,679 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:15:51,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:15:51,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:15:51,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b4d3102, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:15:51,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:15:51,681 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:51,681 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:15:51,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d34543, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,683 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:15:51,683 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:15:51,684 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:15:51,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:15:51,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:15:51,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13430371, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:15:51,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:15:51,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:51,685 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:15:51,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29fc64f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,687 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:15:51,687 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:15:51,688 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:15:51,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:15:51,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:15:51,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62bed90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:15:51,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:15:51,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:51,690 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:15:51,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d4fe6b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c58aaf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,691 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6b1531871d35,33843,-1 for getting cluster id 2024-11-19T00:15:51,691 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T00:15:51,692 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0b318443-22f9-457d-92e4-12357fd10a8d' 2024-11-19T00:15:51,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T00:15:51,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0b318443-22f9-457d-92e4-12357fd10a8d" 2024-11-19T00:15:51,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4b30e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6b1531871d35,33843,-1] 2024-11-19T00:15:51,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T00:15:51,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:15:51,699 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T00:15:51,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6670e249, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T00:15:51,704 DEBUG [Thread-2425 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:15:51,704 DEBUG [Thread-2429 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:15:51,705 DEBUG [Thread-2423 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:15:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:15:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:15:51,705 DEBUG [Thread-2420 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:15:51,705 DEBUG [Thread-2414 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:15:51,706 DEBUG [Thread-2418 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:15:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:15:51,706 DEBUG [Thread-2420 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:15:51,706 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:15:51,706 DEBUG [Thread-2420 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:15:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:15:51,707 DEBUG [Thread-2414 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:15:51,707 DEBUG [Thread-2414 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:15:51,708 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:15:51,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:15:51,708 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:15:51,708 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:15:51,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:15:51,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:15:51,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:15:51,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:15:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-19T00:15:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:15:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:15:51,711 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:15:51,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:15:51,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:15:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-19T00:15:51,712 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:15:51,713 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:15:51,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:15:51,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:15:51,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:15:51,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:15:51,721 DEBUG [Thread-2427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:15:51,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:15:51,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:15:51,724 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:15:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:15:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:51,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfb45e8932b673e843a8031625dfee57 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T00:15:51,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=A 2024-11-19T00:15:51,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:51,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=B 2024-11-19T00:15:51,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:51,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=C 2024-11-19T00:15:51,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:51,732 DEBUG [Thread-2416 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:15:51,732 DEBUG [Thread-2412 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:15:51,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:15:51,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:15:51,733 DEBUG [Thread-2431 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T00:15:51,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:15:51,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:15:51,739 DEBUG [Thread-2416 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:15:51,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6b1531871d35,38567,1731975194317, seqNum=-1] 2024-11-19T00:15:51,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:15:51,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:15:51,746 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:15:51,754 DEBUG [Thread-2416 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T00:15:51,756 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T00:15:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5] 2024-11-19T00:15:51,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975351749, max timestamp=1731975351770Num uniques 859; 2024-11-19T00:15:51,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975351749, max timestamp=1731975351770Num uniques 859; 2024-11-19T00:15:51,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:15:51,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:15:51,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975351749, max timestamp=1731975351770Num uniques 859; 2024-11-19T00:15:51,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:15:51,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119426c084a08a84cd8a342a34857ed916f_cfb45e8932b673e843a8031625dfee57 is 50, key is test_row_0/A:col10/1731975351720/Put/seqid=0 2024-11-19T00:15:51,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742366_1542 (size=12154) 2024-11-19T00:15:51,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731975351792, max timestamp=1731975351803Num uniques 1018; 2024-11-19T00:15:51,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-19T00:15:51,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731975351792, max timestamp=1731975351803Num uniques 1018; 2024-11-19T00:15:51,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T00:15:51,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731975351792, max timestamp=1731975351803Num uniques 1018; 2024-11-19T00:15:51,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T00:15:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-19T00:15:51,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T00:15:51,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:51,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:15:51,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:51,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:51,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:51,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975411883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:51,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975411883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:51,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:51,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:51,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:51,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975411884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:51,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:51,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:51,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:51,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975411885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:51,893 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:51,893 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:51,893 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:51,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975411891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:51,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:51,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975411994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:51,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:51,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:51,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:52,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975412004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:52,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:52,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975412004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:52,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975412005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:52,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975412005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:52,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T00:15:52,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:52,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:15:52,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:52,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:52,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:52,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:52,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-19T00:15:52,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T00:15:52,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:52,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:15:52,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:52,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:52,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:52,203 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:52,208 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119426c084a08a84cd8a342a34857ed916f_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119426c084a08a84cd8a342a34857ed916f_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:52,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975412204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:52,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:52,209 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/c6702a6160094448b3ae3cc863a651a3, store: [table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:15:52,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/c6702a6160094448b3ae3cc863a651a3 is 175, key is test_row_0/A:col10/1731975351720/Put/seqid=0 2024-11-19T00:15:52,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975412215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:52,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975412215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:52,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975412215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975412215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:52,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742367_1543 (size=30955) 2024-11-19T00:15:52,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T00:15:52,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:52,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:15:52,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:52,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:52,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-19T00:15:52,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T00:15:52,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:52,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:15:52,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:52,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:52,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:52,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975412513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:52,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975412524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:52,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:52,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975412524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:52,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:52,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975412524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:52,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975412524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:52,639 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/c6702a6160094448b3ae3cc863a651a3 2024-11-19T00:15:52,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T00:15:52,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:52,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:15:52,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:52,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:52,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:52,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/e57ebb7dd37a4a5ab3cd2c9c78a5be44 is 50, key is test_row_0/B:col10/1731975351720/Put/seqid=0 2024-11-19T00:15:52,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742368_1544 (size=12001) 2024-11-19T00:15:52,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/e57ebb7dd37a4a5ab3cd2c9c78a5be44 2024-11-19T00:15:52,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/517f0ffcf49f48f5840ba79b8dc9ab27 is 50, key is test_row_0/C:col10/1731975351720/Put/seqid=0 2024-11-19T00:15:52,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742369_1545 (size=12001) 2024-11-19T00:15:52,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T00:15:52,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:52,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:15:52,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:52,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:52,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-19T00:15:52,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T00:15:52,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:52,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:15:52,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:52,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:52,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:53,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:53,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975413024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:53,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:53,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:53,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:53,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:53,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975413034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:53,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:53,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:53,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:53,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975413034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:53,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:53,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:53,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975413034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:53,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975413034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:53,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T00:15:53,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:53,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:15:53,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:53,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:53,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:53,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/517f0ffcf49f48f5840ba79b8dc9ab27 2024-11-19T00:15:53,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/c6702a6160094448b3ae3cc863a651a3 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/c6702a6160094448b3ae3cc863a651a3 2024-11-19T00:15:53,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/c6702a6160094448b3ae3cc863a651a3, entries=150, sequenceid=14, filesize=30.2 K 2024-11-19T00:15:53,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/e57ebb7dd37a4a5ab3cd2c9c78a5be44 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/e57ebb7dd37a4a5ab3cd2c9c78a5be44 2024-11-19T00:15:53,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/e57ebb7dd37a4a5ab3cd2c9c78a5be44, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T00:15:53,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/517f0ffcf49f48f5840ba79b8dc9ab27 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/517f0ffcf49f48f5840ba79b8dc9ab27 2024-11-19T00:15:53,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/517f0ffcf49f48f5840ba79b8dc9ab27, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T00:15:53,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for cfb45e8932b673e843a8031625dfee57 in 1404ms, sequenceid=14, compaction requested=false 2024-11-19T00:15:53,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:15:53,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T00:15:53,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:53,254 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing cfb45e8932b673e843a8031625dfee57 3/3 column families, dataSize=134.18 KB heapSize=379.45 KB 2024-11-19T00:15:53,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=A 2024-11-19T00:15:53,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:53,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=B 2024-11-19T00:15:53,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:53,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=C 2024-11-19T00:15:53,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:53,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119830e7ffdfe844b4c9b461713f8daa98a_cfb45e8932b673e843a8031625dfee57 is 50, key is test_row_0/A:col10/1731975351888/Put/seqid=0 2024-11-19T00:15:53,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742370_1546 (size=12154) 2024-11-19T00:15:53,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:53,667 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119830e7ffdfe844b4c9b461713f8daa98a_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119830e7ffdfe844b4c9b461713f8daa98a_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:53,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/6527b32b8058494697649eb7b867a717, store: [table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:15:53,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/6527b32b8058494697649eb7b867a717 is 175, key is test_row_0/A:col10/1731975351888/Put/seqid=0 2024-11-19T00:15:53,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742371_1547 (size=30955) 2024-11-19T00:15:53,700 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=67, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/6527b32b8058494697649eb7b867a717 2024-11-19T00:15:53,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/caef8f6b5f2249738618063c12dfec31 is 50, key is test_row_0/B:col10/1731975351888/Put/seqid=0 2024-11-19T00:15:53,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742372_1548 (size=12001) 2024-11-19T00:15:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-19T00:15:53,981 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T00:15:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:54,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:15:54,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975414054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:54,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:54,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:54,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975414055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975414056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:54,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:54,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975414064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:54,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975414064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:54,116 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/caef8f6b5f2249738618063c12dfec31 2024-11-19T00:15:54,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/98cbbb1caa6f432ab46a6325ee16453c is 50, key is test_row_0/C:col10/1731975351888/Put/seqid=0 2024-11-19T00:15:54,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742373_1549 (size=12001) 2024-11-19T00:15:54,141 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/98cbbb1caa6f432ab46a6325ee16453c 2024-11-19T00:15:54,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/6527b32b8058494697649eb7b867a717 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/6527b32b8058494697649eb7b867a717 2024-11-19T00:15:54,155 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/6527b32b8058494697649eb7b867a717, entries=150, sequenceid=67, filesize=30.2 K 2024-11-19T00:15:54,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/caef8f6b5f2249738618063c12dfec31 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/caef8f6b5f2249738618063c12dfec31 2024-11-19T00:15:54,161 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/caef8f6b5f2249738618063c12dfec31, entries=150, sequenceid=67, filesize=11.7 K 2024-11-19T00:15:54,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/98cbbb1caa6f432ab46a6325ee16453c as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/98cbbb1caa6f432ab46a6325ee16453c 2024-11-19T00:15:54,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975414175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:54,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975414175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:54,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:54,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,176 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/98cbbb1caa6f432ab46a6325ee16453c, entries=150, sequenceid=67, filesize=11.7 K 2024-11-19T00:15:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975414174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:54,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975414175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:54,177 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~379.41 KB/388512, currentSize=46.96 KB/48090 for cfb45e8932b673e843a8031625dfee57 in 922ms, sequenceid=67, compaction requested=false 2024-11-19T00:15:54,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:15:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-19T00:15:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-19T00:15:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:54,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfb45e8932b673e843a8031625dfee57 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T00:15:54,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=A 2024-11-19T00:15:54,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:54,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=B 2024-11-19T00:15:54,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:54,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=C 2024-11-19T00:15:54,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:54,180 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-19T00:15:54,180 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4650 sec 2024-11-19T00:15:54,182 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.4720 sec 2024-11-19T00:15:54,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c2a881cf354c434682dc9b632cc02595_cfb45e8932b673e843a8031625dfee57 is 50, key is test_row_0/A:col10/1731975354176/Put/seqid=0 2024-11-19T00:15:54,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742374_1550 (size=14594) 2024-11-19T00:15:54,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975354283, max timestamp=1731975354316Num uniques 859; 2024-11-19T00:15:54,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975354283, max timestamp=1731975354316Num uniques 859; 2024-11-19T00:15:54,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:15:54,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:15:54,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975354283, max timestamp=1731975354316Num uniques 859; 2024-11-19T00:15:54,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:15:54,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975414384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:54,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:54,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975414386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:54,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:54,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975414390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975414390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:54,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975414391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:54,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975414494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:54,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975414504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:54,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:54,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975414504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:54,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:54,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:54,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975414504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:54,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:54,605 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:54,607 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c2a881cf354c434682dc9b632cc02595_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c2a881cf354c434682dc9b632cc02595_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:54,609 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/e219f363fdda48f289066a2f43e7d0c4, store: [table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:15:54,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/e219f363fdda48f289066a2f43e7d0c4 is 175, key is test_row_0/A:col10/1731975354176/Put/seqid=0 2024-11-19T00:15:54,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742375_1551 (size=39549) 2024-11-19T00:15:54,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975414704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:54,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:54,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975414704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:54,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:54,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975414713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:54,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:54,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975414713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:54,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:54,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975414713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:55,013 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/e219f363fdda48f289066a2f43e7d0c4 2024-11-19T00:15:55,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975415014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:55,016 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,016 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,016 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:55,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/f16e70a408784f58abe2c46a2a294d57 is 50, key is test_row_0/B:col10/1731975354176/Put/seqid=0 2024-11-19T00:15:55,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742376_1552 (size=12001) 2024-11-19T00:15:55,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/f16e70a408784f58abe2c46a2a294d57 2024-11-19T00:15:55,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:55,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975415023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975415023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:55,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:55,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975415024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:55,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:55,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/06f285967b1846bd9f03d9f788dec337 is 50, key is test_row_0/C:col10/1731975354176/Put/seqid=0 2024-11-19T00:15:55,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742377_1553 (size=12001) 2024-11-19T00:15:55,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/06f285967b1846bd9f03d9f788dec337 2024-11-19T00:15:55,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/e219f363fdda48f289066a2f43e7d0c4 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/e219f363fdda48f289066a2f43e7d0c4 2024-11-19T00:15:55,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/e219f363fdda48f289066a2f43e7d0c4, entries=200, sequenceid=78, filesize=38.6 K 2024-11-19T00:15:55,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/f16e70a408784f58abe2c46a2a294d57 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/f16e70a408784f58abe2c46a2a294d57 2024-11-19T00:15:55,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/f16e70a408784f58abe2c46a2a294d57, entries=150, sequenceid=78, filesize=11.7 K 2024-11-19T00:15:55,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/06f285967b1846bd9f03d9f788dec337 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/06f285967b1846bd9f03d9f788dec337 2024-11-19T00:15:55,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/06f285967b1846bd9f03d9f788dec337, entries=150, sequenceid=78, filesize=11.7 K 2024-11-19T00:15:55,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for cfb45e8932b673e843a8031625dfee57 in 879ms, sequenceid=78, compaction requested=true 2024-11-19T00:15:55,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:15:55,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb45e8932b673e843a8031625dfee57:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:15:55,056 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:55,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:55,056 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:55,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb45e8932b673e843a8031625dfee57:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:15:55,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:55,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb45e8932b673e843a8031625dfee57:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:15:55,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:55,060 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:55,060 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:55,060 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): cfb45e8932b673e843a8031625dfee57/A is initiating minor compaction (all files) 2024-11-19T00:15:55,060 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): cfb45e8932b673e843a8031625dfee57/B is initiating minor compaction (all files) 2024-11-19T00:15:55,060 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb45e8932b673e843a8031625dfee57/A in TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:55,060 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb45e8932b673e843a8031625dfee57/B in TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:55,060 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/c6702a6160094448b3ae3cc863a651a3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/6527b32b8058494697649eb7b867a717, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/e219f363fdda48f289066a2f43e7d0c4] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp, totalSize=99.1 K 2024-11-19T00:15:55,061 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:55,061 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. files: [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/c6702a6160094448b3ae3cc863a651a3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/6527b32b8058494697649eb7b867a717, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/e219f363fdda48f289066a2f43e7d0c4] 2024-11-19T00:15:55,061 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/e57ebb7dd37a4a5ab3cd2c9c78a5be44, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/caef8f6b5f2249738618063c12dfec31, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/f16e70a408784f58abe2c46a2a294d57] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp, totalSize=35.2 K 2024-11-19T00:15:55,062 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting e57ebb7dd37a4a5ab3cd2c9c78a5be44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731975351720 2024-11-19T00:15:55,062 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6702a6160094448b3ae3cc863a651a3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731975351720 2024-11-19T00:15:55,063 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting caef8f6b5f2249738618063c12dfec31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731975351828 2024-11-19T00:15:55,064 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting f16e70a408784f58abe2c46a2a294d57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731975354054 2024-11-19T00:15:55,064 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6527b32b8058494697649eb7b867a717, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731975351828 2024-11-19T00:15:55,064 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting e219f363fdda48f289066a2f43e7d0c4, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731975354054 2024-11-19T00:15:55,070 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:15:55,072 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb45e8932b673e843a8031625dfee57#B#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:55,073 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/2db83489d4364853ac02c9c14beda29d is 50, key is test_row_0/B:col10/1731975354176/Put/seqid=0 2024-11-19T00:15:55,074 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111903d3019f17a3420da7e1b0aaaeb0fdb0_cfb45e8932b673e843a8031625dfee57 store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:15:55,078 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111903d3019f17a3420da7e1b0aaaeb0fdb0_cfb45e8932b673e843a8031625dfee57, store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:15:55,078 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111903d3019f17a3420da7e1b0aaaeb0fdb0_cfb45e8932b673e843a8031625dfee57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:15:55,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742378_1554 (size=12104) 2024-11-19T00:15:55,098 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/2db83489d4364853ac02c9c14beda29d as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/2db83489d4364853ac02c9c14beda29d 2024-11-19T00:15:55,106 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb45e8932b673e843a8031625dfee57/B of cfb45e8932b673e843a8031625dfee57 into 2db83489d4364853ac02c9c14beda29d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:55,106 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:15:55,106 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., storeName=cfb45e8932b673e843a8031625dfee57/B, priority=13, startTime=1731975355056; duration=0sec 2024-11-19T00:15:55,106 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:55,106 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb45e8932b673e843a8031625dfee57:B 2024-11-19T00:15:55,106 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:55,108 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:55,108 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): cfb45e8932b673e843a8031625dfee57/C is initiating minor compaction (all files) 2024-11-19T00:15:55,108 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb45e8932b673e843a8031625dfee57/C in TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:55,108 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/517f0ffcf49f48f5840ba79b8dc9ab27, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/98cbbb1caa6f432ab46a6325ee16453c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/06f285967b1846bd9f03d9f788dec337] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp, totalSize=35.2 K 2024-11-19T00:15:55,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742379_1555 (size=4469) 2024-11-19T00:15:55,108 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 517f0ffcf49f48f5840ba79b8dc9ab27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731975351720 2024-11-19T00:15:55,109 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 98cbbb1caa6f432ab46a6325ee16453c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731975351828 2024-11-19T00:15:55,109 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 06f285967b1846bd9f03d9f788dec337, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731975354054 2024-11-19T00:15:55,110 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb45e8932b673e843a8031625dfee57#A#compaction#472 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:55,111 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/1e7c0d1525a74bd5b74b792c086f967d is 175, key is test_row_0/A:col10/1731975354176/Put/seqid=0 2024-11-19T00:15:55,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742380_1556 (size=31058) 2024-11-19T00:15:55,117 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb45e8932b673e843a8031625dfee57#C#compaction#473 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:55,118 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/caf69cfd321341de890c9d94dc88b14b is 50, key is test_row_0/C:col10/1731975354176/Put/seqid=0 2024-11-19T00:15:55,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742381_1557 (size=12104) 2024-11-19T00:15:55,136 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/caf69cfd321341de890c9d94dc88b14b as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/caf69cfd321341de890c9d94dc88b14b 2024-11-19T00:15:55,141 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb45e8932b673e843a8031625dfee57/C of cfb45e8932b673e843a8031625dfee57 into caf69cfd321341de890c9d94dc88b14b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:55,141 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:15:55,141 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., storeName=cfb45e8932b673e843a8031625dfee57/C, priority=13, startTime=1731975355056; duration=0sec 2024-11-19T00:15:55,141 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:55,141 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb45e8932b673e843a8031625dfee57:C 2024-11-19T00:15:55,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfb45e8932b673e843a8031625dfee57 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-19T00:15:55,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=A 2024-11-19T00:15:55,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:15:55,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=B 2024-11-19T00:15:55,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:15:55,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=C 2024-11-19T00:15:55,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T00:15:55,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:55,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119938fa31dc976423bb76a3896226e1de4_cfb45e8932b673e843a8031625dfee57 is 50, key is test_row_0/A:col10/1731975354385/Put/seqid=0 2024-11-19T00:15:55,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742382_1558 (size=12154) 2024-11-19T00:15:55,259 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:55,264 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119938fa31dc976423bb76a3896226e1de4_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119938fa31dc976423bb76a3896226e1de4_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:55,264 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/8f4d0a52941a40a79a4f02504077e6db, store: [table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:15:55,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/8f4d0a52941a40a79a4f02504077e6db is 175, key is test_row_0/A:col10/1731975354385/Put/seqid=0 2024-11-19T00:15:55,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:55,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975415263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:55,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:55,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742383_1559 (size=30955) 2024-11-19T00:15:55,276 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/8f4d0a52941a40a79a4f02504077e6db 2024-11-19T00:15:55,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/4467b17fa645498f99303e590e0ccae1 is 50, key is test_row_0/B:col10/1731975354385/Put/seqid=0 2024-11-19T00:15:55,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742384_1560 (size=12001) 2024-11-19T00:15:55,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/4467b17fa645498f99303e590e0ccae1 2024-11-19T00:15:55,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/3161c6d98a264b9a94c2841ac10bd340 is 50, key is test_row_0/C:col10/1731975354385/Put/seqid=0 2024-11-19T00:15:55,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742385_1561 (size=12001) 2024-11-19T00:15:55,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/3161c6d98a264b9a94c2841ac10bd340 2024-11-19T00:15:55,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:55,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975415374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:55,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:55,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/8f4d0a52941a40a79a4f02504077e6db as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/8f4d0a52941a40a79a4f02504077e6db 2024-11-19T00:15:55,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/8f4d0a52941a40a79a4f02504077e6db, entries=150, sequenceid=118, filesize=30.2 K 2024-11-19T00:15:55,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/4467b17fa645498f99303e590e0ccae1 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/4467b17fa645498f99303e590e0ccae1 2024-11-19T00:15:55,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/4467b17fa645498f99303e590e0ccae1, entries=150, sequenceid=118, filesize=11.7 K 2024-11-19T00:15:55,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/3161c6d98a264b9a94c2841ac10bd340 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/3161c6d98a264b9a94c2841ac10bd340 2024-11-19T00:15:55,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/3161c6d98a264b9a94c2841ac10bd340, entries=150, sequenceid=118, filesize=11.7 K 2024-11-19T00:15:55,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for cfb45e8932b673e843a8031625dfee57 in 172ms, sequenceid=118, compaction requested=false 2024-11-19T00:15:55,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:15:55,519 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/1e7c0d1525a74bd5b74b792c086f967d as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/1e7c0d1525a74bd5b74b792c086f967d 2024-11-19T00:15:55,525 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb45e8932b673e843a8031625dfee57/A of cfb45e8932b673e843a8031625dfee57 into 1e7c0d1525a74bd5b74b792c086f967d(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:55,525 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:15:55,525 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., storeName=cfb45e8932b673e843a8031625dfee57/A, priority=13, startTime=1731975355056; duration=0sec 2024-11-19T00:15:55,526 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:55,526 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb45e8932b673e843a8031625dfee57:A 2024-11-19T00:15:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:55,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfb45e8932b673e843a8031625dfee57 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T00:15:55,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=A 2024-11-19T00:15:55,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:55,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=B 2024-11-19T00:15:55,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:55,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=C 2024-11-19T00:15:55,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:55,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ddb17f701c4c4055b228930804747d4e_cfb45e8932b673e843a8031625dfee57 is 50, key is test_row_0/A:col10/1731975355255/Put/seqid=0 2024-11-19T00:15:55,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742386_1562 (size=14644) 2024-11-19T00:15:55,568 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:55,571 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ddb17f701c4c4055b228930804747d4e_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ddb17f701c4c4055b228930804747d4e_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:55,572 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/4c82092a39774e4791b9ff0a1ad9c097, store: [table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:15:55,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/4c82092a39774e4791b9ff0a1ad9c097 is 175, key is test_row_0/A:col10/1731975355255/Put/seqid=0 2024-11-19T00:15:55,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975355561, max timestamp=1731975355576Num uniques 859; 2024-11-19T00:15:55,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:15:55,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975355561, max timestamp=1731975355576Num uniques 859; 2024-11-19T00:15:55,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:15:55,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975355561, max timestamp=1731975355576Num uniques 859; 2024-11-19T00:15:55,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:15:55,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742387_1563 (size=39599) 2024-11-19T00:15:55,588 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/4c82092a39774e4791b9ff0a1ad9c097 2024-11-19T00:15:55,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/e335deb159984ffa9f0ddcdeefceb734 is 50, key is test_row_0/B:col10/1731975355255/Put/seqid=0 2024-11-19T00:15:55,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742388_1564 (size=12051) 2024-11-19T00:15:55,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975415604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:55,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:55,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975415653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:55,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975415653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:55,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975415658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:55,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:55,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975415660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:55,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:55,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975415764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:55,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:55,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975415773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:55,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:55,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975415773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:55,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:55,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975415773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:55,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:55,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975415774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:55,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:55,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-19T00:15:55,884 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:15:55,884 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:15:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-19T00:15:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T00:15:55,886 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:15:55,886 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:15:55,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:15:55,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975415973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:55,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975415984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:55,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:55,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975415984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:55,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:55,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975415984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:55,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:55,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975415984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:55,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:55,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T00:15:56,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/e335deb159984ffa9f0ddcdeefceb734 2024-11-19T00:15:56,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/ffb8bd9d0211451b8076d447539acc93 is 50, key is test_row_0/C:col10/1731975355255/Put/seqid=0 2024-11-19T00:15:56,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742389_1565 (size=12051) 2024-11-19T00:15:56,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T00:15:56,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:56,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:15:56,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:56,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:56,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:56,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T00:15:56,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:56,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:15:56,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:56,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:56,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T00:15:56,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975416284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:56,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:56,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975416294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:56,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975416294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:56,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975416294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:56,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:56,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975416294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:56,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:56,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T00:15:56,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:56,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:15:56,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:56,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:56,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:56,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/ffb8bd9d0211451b8076d447539acc93 2024-11-19T00:15:56,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/4c82092a39774e4791b9ff0a1ad9c097 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/4c82092a39774e4791b9ff0a1ad9c097 2024-11-19T00:15:56,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/4c82092a39774e4791b9ff0a1ad9c097, entries=200, sequenceid=129, filesize=38.7 K 2024-11-19T00:15:56,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/e335deb159984ffa9f0ddcdeefceb734 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/e335deb159984ffa9f0ddcdeefceb734 2024-11-19T00:15:56,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/e335deb159984ffa9f0ddcdeefceb734, entries=150, sequenceid=129, filesize=11.8 K 2024-11-19T00:15:56,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/ffb8bd9d0211451b8076d447539acc93 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/ffb8bd9d0211451b8076d447539acc93 2024-11-19T00:15:56,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/ffb8bd9d0211451b8076d447539acc93, entries=150, sequenceid=129, filesize=11.8 K 2024-11-19T00:15:56,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for cfb45e8932b673e843a8031625dfee57 in 914ms, sequenceid=129, compaction requested=true 2024-11-19T00:15:56,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:15:56,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb45e8932b673e843a8031625dfee57:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:15:56,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:56,443 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:56,443 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:56,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb45e8932b673e843a8031625dfee57:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:15:56,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:56,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb45e8932b673e843a8031625dfee57:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:15:56,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:56,444 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36156 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:56,444 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): cfb45e8932b673e843a8031625dfee57/B is initiating minor compaction (all files) 2024-11-19T00:15:56,444 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb45e8932b673e843a8031625dfee57/B in TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:56,444 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/2db83489d4364853ac02c9c14beda29d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/4467b17fa645498f99303e590e0ccae1, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/e335deb159984ffa9f0ddcdeefceb734] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp, totalSize=35.3 K 2024-11-19T00:15:56,446 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101612 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:56,446 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 2db83489d4364853ac02c9c14beda29d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731975354054 2024-11-19T00:15:56,446 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): cfb45e8932b673e843a8031625dfee57/A is initiating minor compaction (all files) 2024-11-19T00:15:56,446 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb45e8932b673e843a8031625dfee57/A in TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:56,446 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/1e7c0d1525a74bd5b74b792c086f967d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/8f4d0a52941a40a79a4f02504077e6db, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/4c82092a39774e4791b9ff0a1ad9c097] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp, totalSize=99.2 K 2024-11-19T00:15:56,447 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:56,447 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 4467b17fa645498f99303e590e0ccae1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731975354385 2024-11-19T00:15:56,447 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. files: [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/1e7c0d1525a74bd5b74b792c086f967d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/8f4d0a52941a40a79a4f02504077e6db, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/4c82092a39774e4791b9ff0a1ad9c097] 2024-11-19T00:15:56,447 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting e335deb159984ffa9f0ddcdeefceb734, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731975355241 2024-11-19T00:15:56,447 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e7c0d1525a74bd5b74b792c086f967d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731975354054 2024-11-19T00:15:56,447 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f4d0a52941a40a79a4f02504077e6db, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731975354385 2024-11-19T00:15:56,448 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c82092a39774e4791b9ff0a1ad9c097, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731975355241 2024-11-19T00:15:56,458 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:15:56,463 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb45e8932b673e843a8031625dfee57#B#compaction#481 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:56,464 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/f41640f6e4dd446fbdbab1488b3241fe is 50, key is test_row_0/B:col10/1731975355255/Put/seqid=0 2024-11-19T00:15:56,466 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411191c9e3d049bcc4f6f895db8c15daf0c6a_cfb45e8932b673e843a8031625dfee57 store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:15:56,468 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411191c9e3d049bcc4f6f895db8c15daf0c6a_cfb45e8932b673e843a8031625dfee57, store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:15:56,468 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191c9e3d049bcc4f6f895db8c15daf0c6a_cfb45e8932b673e843a8031625dfee57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:15:56,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742390_1566 (size=12257) 2024-11-19T00:15:56,498 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/f41640f6e4dd446fbdbab1488b3241fe as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/f41640f6e4dd446fbdbab1488b3241fe 2024-11-19T00:15:56,503 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb45e8932b673e843a8031625dfee57/B of cfb45e8932b673e843a8031625dfee57 into f41640f6e4dd446fbdbab1488b3241fe(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:56,503 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:15:56,503 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., storeName=cfb45e8932b673e843a8031625dfee57/B, priority=13, startTime=1731975356443; duration=0sec 2024-11-19T00:15:56,503 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:56,503 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb45e8932b673e843a8031625dfee57:B 2024-11-19T00:15:56,503 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:56,504 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36156 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:56,504 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): cfb45e8932b673e843a8031625dfee57/C is initiating minor compaction (all files) 2024-11-19T00:15:56,504 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb45e8932b673e843a8031625dfee57/C in TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:56,505 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/caf69cfd321341de890c9d94dc88b14b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/3161c6d98a264b9a94c2841ac10bd340, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/ffb8bd9d0211451b8076d447539acc93] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp, totalSize=35.3 K 2024-11-19T00:15:56,505 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting caf69cfd321341de890c9d94dc88b14b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731975354054 2024-11-19T00:15:56,506 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 3161c6d98a264b9a94c2841ac10bd340, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731975354385 2024-11-19T00:15:56,506 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting ffb8bd9d0211451b8076d447539acc93, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731975355241 2024-11-19T00:15:56,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T00:15:56,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:56,511 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing cfb45e8932b673e843a8031625dfee57 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-19T00:15:56,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=A 2024-11-19T00:15:56,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:56,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=B 2024-11-19T00:15:56,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:56,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=C 2024-11-19T00:15:56,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T00:15:56,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742391_1567 (size=4469) 2024-11-19T00:15:56,528 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb45e8932b673e843a8031625dfee57#C#compaction#482 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:56,528 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/52fd3c6c866749b586bea3441f3b4222 is 50, key is test_row_0/C:col10/1731975355255/Put/seqid=0 2024-11-19T00:15:56,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196b868475e83c4f5aa2e3beb96c9622b4_cfb45e8932b673e843a8031625dfee57 is 50, key is test_row_0/A:col10/1731975355652/Put/seqid=0 2024-11-19T00:15:56,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742392_1568 (size=12257) 2024-11-19T00:15:56,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742393_1569 (size=12304) 2024-11-19T00:15:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:56,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:15:56,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:56,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975416808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:56,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:56,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975416809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:56,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:56,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975416814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:56,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975416814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975416814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:56,918 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb45e8932b673e843a8031625dfee57#A#compaction#480 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:56,918 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/c0d79232a028430aa0bb8a433f1b8e1e is 175, key is test_row_0/A:col10/1731975355255/Put/seqid=0 2024-11-19T00:15:56,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975416923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:56,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975416923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:56,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975416924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:56,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:56,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:56,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742394_1570 (size=31211) 2024-11-19T00:15:56,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:56,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:56,989 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/52fd3c6c866749b586bea3441f3b4222 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/52fd3c6c866749b586bea3441f3b4222 2024-11-19T00:15:56,996 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb45e8932b673e843a8031625dfee57/C of cfb45e8932b673e843a8031625dfee57 into 52fd3c6c866749b586bea3441f3b4222(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:56,996 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:15:56,996 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., storeName=cfb45e8932b673e843a8031625dfee57/C, priority=13, startTime=1731975356443; duration=0sec 2024-11-19T00:15:56,996 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:56,996 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb45e8932b673e843a8031625dfee57:C 2024-11-19T00:15:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T00:15:57,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,032 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196b868475e83c4f5aa2e3beb96c9622b4_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196b868475e83c4f5aa2e3beb96c9622b4_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:57,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/3910737fac7f48528da1388c527ab64c, store: [table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:15:57,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/3910737fac7f48528da1388c527ab64c is 175, key is test_row_0/A:col10/1731975355652/Put/seqid=0 2024-11-19T00:15:57,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742395_1571 (size=31105) 2024-11-19T00:15:57,062 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/3910737fac7f48528da1388c527ab64c 2024-11-19T00:15:57,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/1886c52eb0744db68fd932ba3314096d is 50, key is test_row_0/B:col10/1731975355652/Put/seqid=0 2024-11-19T00:15:57,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742396_1572 (size=12151) 2024-11-19T00:15:57,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975417134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:57,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975417134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:57,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:57,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:57,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:57,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:57,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:57,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:57,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975417143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:57,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:57,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:57,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:57,334 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/c0d79232a028430aa0bb8a433f1b8e1e as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/c0d79232a028430aa0bb8a433f1b8e1e 2024-11-19T00:15:57,337 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb45e8932b673e843a8031625dfee57/A of cfb45e8932b673e843a8031625dfee57 into c0d79232a028430aa0bb8a433f1b8e1e(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:57,337 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:15:57,337 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., storeName=cfb45e8932b673e843a8031625dfee57/A, priority=13, startTime=1731975356443; duration=0sec 2024-11-19T00:15:57,338 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:57,338 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb45e8932b673e843a8031625dfee57:A 2024-11-19T00:15:57,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975417443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:57,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:57,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:57,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:57,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975417443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:57,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975417454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:57,479 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/1886c52eb0744db68fd932ba3314096d 2024-11-19T00:15:57,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/703526b5b1534d11a21ec161bd938781 is 50, key is test_row_0/C:col10/1731975355652/Put/seqid=0 2024-11-19T00:15:57,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742397_1573 (size=12151) 2024-11-19T00:15:57,490 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/703526b5b1534d11a21ec161bd938781 2024-11-19T00:15:57,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/3910737fac7f48528da1388c527ab64c as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/3910737fac7f48528da1388c527ab64c 2024-11-19T00:15:57,499 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/3910737fac7f48528da1388c527ab64c, entries=150, sequenceid=167, filesize=30.4 K 2024-11-19T00:15:57,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/1886c52eb0744db68fd932ba3314096d as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/1886c52eb0744db68fd932ba3314096d 2024-11-19T00:15:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,504 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/1886c52eb0744db68fd932ba3314096d, entries=150, sequenceid=167, filesize=11.9 K 2024-11-19T00:15:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/703526b5b1534d11a21ec161bd938781 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/703526b5b1534d11a21ec161bd938781 2024-11-19T00:15:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,509 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/703526b5b1534d11a21ec161bd938781, entries=150, sequenceid=167, filesize=11.9 K 2024-11-19T00:15:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,510 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for cfb45e8932b673e843a8031625dfee57 in 1000ms, sequenceid=167, compaction requested=false 2024-11-19T00:15:57,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:15:57,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:57,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-19T00:15:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-19T00:15:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-19T00:15:57,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6260 sec 2024-11-19T00:15:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.6300 sec 2024-11-19T00:15:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:57,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfb45e8932b673e843a8031625dfee57 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T00:15:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=A 2024-11-19T00:15:57,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:57,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=B 2024-11-19T00:15:57,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:57,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=C 2024-11-19T00:15:57,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:15:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197e8e58df2fab49f8bd65608e996e0f97_cfb45e8932b673e843a8031625dfee57 is 50, key is test_row_1/A:col10/1731975357842/Put/seqid=0 2024-11-19T00:15:57,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742398_1574 (size=12304) 2024-11-19T00:15:57,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,873 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,878 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197e8e58df2fab49f8bd65608e996e0f97_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197e8e58df2fab49f8bd65608e996e0f97_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:57,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,879 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/806eb1ca16e541ffa05e0e9fc1acfbc2, store: [table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:15:57,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/806eb1ca16e541ffa05e0e9fc1acfbc2 is 175, key is test_row_1/A:col10/1731975357842/Put/seqid=0 2024-11-19T00:15:57,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742399_1575 (size=31101) 2024-11-19T00:15:57,885 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=179, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/806eb1ca16e541ffa05e0e9fc1acfbc2 2024-11-19T00:15:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:57,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/5d1b46bacce445a8a8c94b4f6fbd454d is 50, key is test_row_1/B:col10/1731975357842/Put/seqid=0 2024-11-19T00:15:57,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742400_1576 (size=9757) 2024-11-19T00:15:57,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975357908, max timestamp=1731975357925Num uniques 859; 2024-11-19T00:15:57,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731975357908, max timestamp=1731975357925Num uniques 859; 2024-11-19T00:15:57,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731975357908, max timestamp=1731975357925Num uniques 859; 2024-11-19T00:15:57,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:15:57,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T00:15:57,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6b1531871d35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T00:15:57,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:57,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975417962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:57,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:57,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:57,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975417962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:57,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:57,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:57,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:57,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:57,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:57,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975417962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:57,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:57,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975417963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:57,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:57,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975417968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:57,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:57,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:57,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:58,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T00:15:58,034 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:15:58,035 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:15:58,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-19T00:15:58,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T00:15:58,037 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:15:58,038 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:15:58,038 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:15:58,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:58,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975418074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:58,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:58,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:58,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:58,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:58,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975418074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:58,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:58,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:58,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:58,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:58,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975418074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:58,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:58,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975418083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:58,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:58,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:58,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:58,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T00:15:58,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T00:15:58,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:58,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:15:58,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:58,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:58,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:58,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:58,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:58,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975418284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975418284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:58,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:58,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:58,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:58,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975418284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:58,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:58,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:58,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:58,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:58,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:58,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:58,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975418293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:58,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:58,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:58,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:58,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/5d1b46bacce445a8a8c94b4f6fbd454d 2024-11-19T00:15:58,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/2e052dc056374c4f80c339ab170b147d is 50, key is test_row_1/C:col10/1731975357842/Put/seqid=0 2024-11-19T00:15:58,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742401_1577 (size=9757) 2024-11-19T00:15:58,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T00:15:58,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:58,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:15:58,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:58,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:58,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:58,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T00:15:58,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T00:15:58,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:58,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:15:58,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:58,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:58,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:58,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:58,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:58,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975418594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:58,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:58,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:58,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:58,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:58,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975418594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:58,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:58,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:58,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:58,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:58,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975418594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:58,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:58,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:58,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:58,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:58,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975418603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:58,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T00:15:58,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:58,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:15:58,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:58,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:58,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:15:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T00:15:58,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/2e052dc056374c4f80c339ab170b147d 2024-11-19T00:15:58,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/806eb1ca16e541ffa05e0e9fc1acfbc2 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/806eb1ca16e541ffa05e0e9fc1acfbc2 2024-11-19T00:15:58,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/806eb1ca16e541ffa05e0e9fc1acfbc2, entries=150, sequenceid=179, filesize=30.4 K 2024-11-19T00:15:58,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/5d1b46bacce445a8a8c94b4f6fbd454d as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/5d1b46bacce445a8a8c94b4f6fbd454d 2024-11-19T00:15:58,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/5d1b46bacce445a8a8c94b4f6fbd454d, entries=100, sequenceid=179, filesize=9.5 K 2024-11-19T00:15:58,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/2e052dc056374c4f80c339ab170b147d as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/2e052dc056374c4f80c339ab170b147d 2024-11-19T00:15:58,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/2e052dc056374c4f80c339ab170b147d, entries=100, sequenceid=179, filesize=9.5 K 2024-11-19T00:15:58,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for cfb45e8932b673e843a8031625dfee57 in 900ms, sequenceid=179, compaction requested=true 2024-11-19T00:15:58,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:15:58,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb45e8932b673e843a8031625dfee57:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:15:58,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:58,744 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:58,744 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:58,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb45e8932b673e843a8031625dfee57:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:15:58,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:58,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb45e8932b673e843a8031625dfee57:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:15:58,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:58,745 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34165 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:58,745 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): cfb45e8932b673e843a8031625dfee57/B is initiating minor compaction (all files) 2024-11-19T00:15:58,745 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:58,745 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): cfb45e8932b673e843a8031625dfee57/A is initiating minor compaction (all files) 2024-11-19T00:15:58,745 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb45e8932b673e843a8031625dfee57/B in TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:58,745 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb45e8932b673e843a8031625dfee57/A in TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:58,745 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/f41640f6e4dd446fbdbab1488b3241fe, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/1886c52eb0744db68fd932ba3314096d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/5d1b46bacce445a8a8c94b4f6fbd454d] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp, totalSize=33.4 K 2024-11-19T00:15:58,745 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/c0d79232a028430aa0bb8a433f1b8e1e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/3910737fac7f48528da1388c527ab64c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/806eb1ca16e541ffa05e0e9fc1acfbc2] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp, totalSize=91.2 K 2024-11-19T00:15:58,745 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:58,745 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. files: [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/c0d79232a028430aa0bb8a433f1b8e1e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/3910737fac7f48528da1388c527ab64c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/806eb1ca16e541ffa05e0e9fc1acfbc2] 2024-11-19T00:15:58,745 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting f41640f6e4dd446fbdbab1488b3241fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731975355241 2024-11-19T00:15:58,745 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0d79232a028430aa0bb8a433f1b8e1e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731975355241 2024-11-19T00:15:58,745 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 1886c52eb0744db68fd932ba3314096d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731975355593 2024-11-19T00:15:58,745 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3910737fac7f48528da1388c527ab64c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731975355593 2024-11-19T00:15:58,746 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 806eb1ca16e541ffa05e0e9fc1acfbc2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731975356808 2024-11-19T00:15:58,746 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d1b46bacce445a8a8c94b4f6fbd454d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731975356808 2024-11-19T00:15:58,750 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:15:58,758 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111963d5df14ff2c4ba09b3bd9677007aabf_cfb45e8932b673e843a8031625dfee57 store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:15:58,760 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb45e8932b673e843a8031625dfee57#B#compaction#490 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:58,760 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/89f4893c78f04a71a2821a1ffe3c5635 is 50, key is test_row_0/B:col10/1731975355652/Put/seqid=0 2024-11-19T00:15:58,760 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111963d5df14ff2c4ba09b3bd9677007aabf_cfb45e8932b673e843a8031625dfee57, store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:15:58,761 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111963d5df14ff2c4ba09b3bd9677007aabf_cfb45e8932b673e843a8031625dfee57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:15:58,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742403_1579 (size=4469) 2024-11-19T00:15:58,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742402_1578 (size=12459) 2024-11-19T00:15:58,779 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/89f4893c78f04a71a2821a1ffe3c5635 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/89f4893c78f04a71a2821a1ffe3c5635 2024-11-19T00:15:58,783 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb45e8932b673e843a8031625dfee57/B of cfb45e8932b673e843a8031625dfee57 into 89f4893c78f04a71a2821a1ffe3c5635(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:58,783 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:15:58,783 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., storeName=cfb45e8932b673e843a8031625dfee57/B, priority=13, startTime=1731975358744; duration=0sec 2024-11-19T00:15:58,783 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:15:58,783 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb45e8932b673e843a8031625dfee57:B 2024-11-19T00:15:58,783 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:15:58,784 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34165 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:15:58,784 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): cfb45e8932b673e843a8031625dfee57/C is initiating minor compaction (all files) 2024-11-19T00:15:58,784 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb45e8932b673e843a8031625dfee57/C in TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:58,784 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/52fd3c6c866749b586bea3441f3b4222, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/703526b5b1534d11a21ec161bd938781, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/2e052dc056374c4f80c339ab170b147d] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp, totalSize=33.4 K 2024-11-19T00:15:58,784 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 52fd3c6c866749b586bea3441f3b4222, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731975355241 2024-11-19T00:15:58,785 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 703526b5b1534d11a21ec161bd938781, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731975355593 2024-11-19T00:15:58,785 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e052dc056374c4f80c339ab170b147d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731975356808 2024-11-19T00:15:58,790 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb45e8932b673e843a8031625dfee57#C#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:58,790 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/1ea3196725c74a43b7a7d2766b6d112d is 50, key is test_row_0/C:col10/1731975355652/Put/seqid=0 2024-11-19T00:15:58,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742404_1580 (size=12459) 2024-11-19T00:15:58,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T00:15:58,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:58,807 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing cfb45e8932b673e843a8031625dfee57 3/3 column families, dataSize=131.94 KB heapSize=375.22 KB 2024-11-19T00:15:58,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=A 2024-11-19T00:15:58,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:58,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=B 2024-11-19T00:15:58,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:58,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=C 2024-11-19T00:15:58,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T00:15:58,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199cd8e46276454eb58da1f5c4d0a3fbe9_cfb45e8932b673e843a8031625dfee57 is 50, key is test_row_0/A:col10/1731975357961/Put/seqid=0 2024-11-19T00:15:58,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742405_1581 (size=12304) 2024-11-19T00:15:58,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:15:58,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:59,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:59,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975419007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:59,011 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:59,011 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:59,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:59,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975419103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:59,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:59,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:59,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:59,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:59,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975419103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:59,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:59,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:59,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:59,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:59,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975419103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:59,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:59,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:59,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:59,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:59,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975419114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:59,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:59,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975419124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:59,168 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb45e8932b673e843a8031625dfee57#A#compaction#489 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:15:59,168 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/476e038d3ec8410e9a7952eeda08d1c5 is 175, key is test_row_0/A:col10/1731975355652/Put/seqid=0 2024-11-19T00:15:59,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T00:15:59,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742406_1582 (size=31520) 2024-11-19T00:15:59,203 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/476e038d3ec8410e9a7952eeda08d1c5 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/476e038d3ec8410e9a7952eeda08d1c5 2024-11-19T00:15:59,204 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/1ea3196725c74a43b7a7d2766b6d112d as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/1ea3196725c74a43b7a7d2766b6d112d 2024-11-19T00:15:59,208 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb45e8932b673e843a8031625dfee57/A of cfb45e8932b673e843a8031625dfee57 into 476e038d3ec8410e9a7952eeda08d1c5(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:59,208 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:15:59,208 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., storeName=cfb45e8932b673e843a8031625dfee57/A, priority=13, startTime=1731975358744; duration=0sec 2024-11-19T00:15:59,208 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:59,208 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb45e8932b673e843a8031625dfee57:A 2024-11-19T00:15:59,209 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb45e8932b673e843a8031625dfee57/C of cfb45e8932b673e843a8031625dfee57 into 1ea3196725c74a43b7a7d2766b6d112d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:15:59,209 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:15:59,209 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., storeName=cfb45e8932b673e843a8031625dfee57/C, priority=13, startTime=1731975358744; duration=0sec 2024-11-19T00:15:59,209 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:15:59,209 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb45e8932b673e843a8031625dfee57:C 2024-11-19T00:15:59,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:15:59,230 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199cd8e46276454eb58da1f5c4d0a3fbe9_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199cd8e46276454eb58da1f5c4d0a3fbe9_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:15:59,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/0868cbb3653043d68206ed8f5016571f, store: [table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:15:59,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/0868cbb3653043d68206ed8f5016571f is 175, key is test_row_0/A:col10/1731975357961/Put/seqid=0 2024-11-19T00:15:59,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742407_1583 (size=31105) 2024-11-19T00:15:59,239 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/0868cbb3653043d68206ed8f5016571f 2024-11-19T00:15:59,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/b7a63f8628264a2b87e0be1bcccadaaf is 50, key is test_row_0/B:col10/1731975357961/Put/seqid=0 2024-11-19T00:15:59,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742408_1584 (size=12151) 2024-11-19T00:15:59,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975419323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:59,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:59,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:59,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:59,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:15:59,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975419634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:15:59,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:59,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:15:59,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:15:59,656 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/b7a63f8628264a2b87e0be1bcccadaaf 2024-11-19T00:15:59,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/5501a76530fa4b458f9000781ab95f95 is 50, key is test_row_0/C:col10/1731975357961/Put/seqid=0 2024-11-19T00:15:59,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742409_1585 (size=12151) 2024-11-19T00:15:59,670 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/5501a76530fa4b458f9000781ab95f95 2024-11-19T00:15:59,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/0868cbb3653043d68206ed8f5016571f as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/0868cbb3653043d68206ed8f5016571f 2024-11-19T00:15:59,677 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/0868cbb3653043d68206ed8f5016571f, entries=150, sequenceid=216, filesize=30.4 K 2024-11-19T00:15:59,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/b7a63f8628264a2b87e0be1bcccadaaf as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/b7a63f8628264a2b87e0be1bcccadaaf 2024-11-19T00:15:59,680 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/b7a63f8628264a2b87e0be1bcccadaaf, entries=150, sequenceid=216, filesize=11.9 K 2024-11-19T00:15:59,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/5501a76530fa4b458f9000781ab95f95 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/5501a76530fa4b458f9000781ab95f95 2024-11-19T00:15:59,685 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/5501a76530fa4b458f9000781ab95f95, entries=150, sequenceid=216, filesize=11.9 K 2024-11-19T00:15:59,686 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~375.17 KB/384176, currentSize=46.96 KB/48090 for cfb45e8932b673e843a8031625dfee57 in 879ms, sequenceid=216, compaction requested=false 2024-11-19T00:15:59,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:15:59,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:15:59,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-19T00:15:59,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-19T00:15:59,690 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-19T00:15:59,690 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6500 sec 2024-11-19T00:15:59,691 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.6550 sec 2024-11-19T00:16:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:00,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfb45e8932b673e843a8031625dfee57 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T00:16:00,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=A 2024-11-19T00:16:00,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:00,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=B 2024-11-19T00:16:00,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:00,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=C 2024-11-19T00:16:00,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:00,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d6e45c5fc64145858fb07fe802d4e98e_cfb45e8932b673e843a8031625dfee57 is 50, key is test_row_0/A:col10/1731975360117/Put/seqid=0 2024-11-19T00:16:00,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742410_1586 (size=12304) 2024-11-19T00:16:00,141 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:16:00,144 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d6e45c5fc64145858fb07fe802d4e98e_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d6e45c5fc64145858fb07fe802d4e98e_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:00,145 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/94d0d2ccb9d944c8a485570d9109033b, store: [table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:00,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/94d0d2ccb9d944c8a485570d9109033b is 175, key is test_row_0/A:col10/1731975360117/Put/seqid=0 2024-11-19T00:16:00,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742411_1587 (size=31105) 2024-11-19T00:16:00,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T00:16:00,184 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:16:00,184 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:16:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-19T00:16:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T00:16:00,186 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:16:00,186 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:16:00,186 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:16:00,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975420209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:00,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975420209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:00,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975420211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:00,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975420214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:00,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975420214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T00:16:00,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975420324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975420324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:00,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975420324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:00,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975420324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975420324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:00,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T00:16:00,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:00,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:00,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:00,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:00,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:00,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T00:16:00,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:00,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:00,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:00,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:00,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:00,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:00,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T00:16:00,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975420533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975420533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:00,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975420534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:00,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975420534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975420534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:00,538 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,538 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,538 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:00,555 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/94d0d2ccb9d944c8a485570d9109033b 2024-11-19T00:16:00,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/3f6d723d30e448a6b679d9798f3d9c70 is 50, key is test_row_0/B:col10/1731975360117/Put/seqid=0 2024-11-19T00:16:00,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742412_1588 (size=12151) 2024-11-19T00:16:00,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/3f6d723d30e448a6b679d9798f3d9c70 2024-11-19T00:16:00,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/98e0207a41bf44c3af8ee5b57d2d7cc5 is 50, key is test_row_0/C:col10/1731975360117/Put/seqid=0 2024-11-19T00:16:00,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742413_1589 (size=12151) 2024-11-19T00:16:00,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/98e0207a41bf44c3af8ee5b57d2d7cc5 2024-11-19T00:16:00,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/94d0d2ccb9d944c8a485570d9109033b as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/94d0d2ccb9d944c8a485570d9109033b 2024-11-19T00:16:00,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/94d0d2ccb9d944c8a485570d9109033b, entries=150, sequenceid=230, filesize=30.4 K 2024-11-19T00:16:00,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/3f6d723d30e448a6b679d9798f3d9c70 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/3f6d723d30e448a6b679d9798f3d9c70 2024-11-19T00:16:00,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/3f6d723d30e448a6b679d9798f3d9c70, entries=150, sequenceid=230, filesize=11.9 K 2024-11-19T00:16:00,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/98e0207a41bf44c3af8ee5b57d2d7cc5 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/98e0207a41bf44c3af8ee5b57d2d7cc5 2024-11-19T00:16:00,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/98e0207a41bf44c3af8ee5b57d2d7cc5, entries=150, sequenceid=230, filesize=11.9 K 2024-11-19T00:16:00,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for cfb45e8932b673e843a8031625dfee57 in 496ms, sequenceid=230, compaction requested=true 2024-11-19T00:16:00,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:00,615 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:16:00,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb45e8932b673e843a8031625dfee57:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:16:00,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:16:00,615 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:16:00,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb45e8932b673e843a8031625dfee57:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:16:00,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:16:00,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb45e8932b673e843a8031625dfee57:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:16:00,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:16:00,618 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:16:00,619 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): cfb45e8932b673e843a8031625dfee57/B is initiating minor compaction (all files) 2024-11-19T00:16:00,619 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb45e8932b673e843a8031625dfee57/B in TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:00,619 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/89f4893c78f04a71a2821a1ffe3c5635, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/b7a63f8628264a2b87e0be1bcccadaaf, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/3f6d723d30e448a6b679d9798f3d9c70] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp, totalSize=35.9 K 2024-11-19T00:16:00,619 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93730 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:16:00,619 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): cfb45e8932b673e843a8031625dfee57/A is initiating minor compaction (all files) 2024-11-19T00:16:00,619 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb45e8932b673e843a8031625dfee57/A in TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:00,619 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 89f4893c78f04a71a2821a1ffe3c5635, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731975355652 2024-11-19T00:16:00,619 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/476e038d3ec8410e9a7952eeda08d1c5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/0868cbb3653043d68206ed8f5016571f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/94d0d2ccb9d944c8a485570d9109033b] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp, totalSize=91.5 K 2024-11-19T00:16:00,619 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:00,619 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. files: [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/476e038d3ec8410e9a7952eeda08d1c5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/0868cbb3653043d68206ed8f5016571f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/94d0d2ccb9d944c8a485570d9109033b] 2024-11-19T00:16:00,620 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 476e038d3ec8410e9a7952eeda08d1c5, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731975355652 2024-11-19T00:16:00,620 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting b7a63f8628264a2b87e0be1bcccadaaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731975357960 2024-11-19T00:16:00,620 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0868cbb3653043d68206ed8f5016571f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731975357960 2024-11-19T00:16:00,620 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f6d723d30e448a6b679d9798f3d9c70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731975358996 2024-11-19T00:16:00,621 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94d0d2ccb9d944c8a485570d9109033b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731975358996 2024-11-19T00:16:00,631 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:00,633 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb45e8932b673e843a8031625dfee57#B#compaction#499 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:16:00,634 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/26ae2b53980c4689ad02d099da2dc059 is 50, key is test_row_0/B:col10/1731975360117/Put/seqid=0 2024-11-19T00:16:00,634 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111956d8a08c7f8a4754a641de04800a8f49_cfb45e8932b673e843a8031625dfee57 store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:00,635 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111956d8a08c7f8a4754a641de04800a8f49_cfb45e8932b673e843a8031625dfee57, store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:00,635 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111956d8a08c7f8a4754a641de04800a8f49_cfb45e8932b673e843a8031625dfee57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:00,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742414_1590 (size=12561) 2024-11-19T00:16:00,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T00:16:00,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:00,644 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing cfb45e8932b673e843a8031625dfee57 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T00:16:00,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=A 2024-11-19T00:16:00,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:00,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=B 2024-11-19T00:16:00,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:00,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=C 2024-11-19T00:16:00,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:00,645 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/26ae2b53980c4689ad02d099da2dc059 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/26ae2b53980c4689ad02d099da2dc059 2024-11-19T00:16:00,650 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb45e8932b673e843a8031625dfee57/B of cfb45e8932b673e843a8031625dfee57 into 26ae2b53980c4689ad02d099da2dc059(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:16:00,650 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:00,651 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., storeName=cfb45e8932b673e843a8031625dfee57/B, priority=13, startTime=1731975360615; duration=0sec 2024-11-19T00:16:00,651 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:16:00,651 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb45e8932b673e843a8031625dfee57:B 2024-11-19T00:16:00,651 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:16:00,652 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:16:00,652 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): cfb45e8932b673e843a8031625dfee57/C is initiating minor compaction (all files) 2024-11-19T00:16:00,652 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb45e8932b673e843a8031625dfee57/C in TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:00,652 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/1ea3196725c74a43b7a7d2766b6d112d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/5501a76530fa4b458f9000781ab95f95, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/98e0207a41bf44c3af8ee5b57d2d7cc5] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp, totalSize=35.9 K 2024-11-19T00:16:00,652 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ea3196725c74a43b7a7d2766b6d112d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731975355652 2024-11-19T00:16:00,652 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 5501a76530fa4b458f9000781ab95f95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731975357960 2024-11-19T00:16:00,652 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 98e0207a41bf44c3af8ee5b57d2d7cc5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731975358996 2024-11-19T00:16:00,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199c74d37128914710a96771d920d95491_cfb45e8932b673e843a8031625dfee57 is 50, key is test_row_0/A:col10/1731975360212/Put/seqid=0 2024-11-19T00:16:00,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742415_1591 (size=4469) 2024-11-19T00:16:00,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742416_1592 (size=12304) 2024-11-19T00:16:00,668 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb45e8932b673e843a8031625dfee57#C#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:16:00,669 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/879f934daa6f4b5cb1a1fdb3758c0c25 is 50, key is test_row_0/C:col10/1731975360117/Put/seqid=0 2024-11-19T00:16:00,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:16:00,675 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199c74d37128914710a96771d920d95491_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199c74d37128914710a96771d920d95491_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:00,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/20b731532e4247948d1d2be50f85bd1e, store: [table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:00,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/20b731532e4247948d1d2be50f85bd1e is 175, key is test_row_0/A:col10/1731975360212/Put/seqid=0 2024-11-19T00:16:00,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742418_1594 (size=31105) 2024-11-19T00:16:00,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742417_1593 (size=12561) 2024-11-19T00:16:00,696 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/879f934daa6f4b5cb1a1fdb3758c0c25 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/879f934daa6f4b5cb1a1fdb3758c0c25 2024-11-19T00:16:00,701 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb45e8932b673e843a8031625dfee57/C of cfb45e8932b673e843a8031625dfee57 into 879f934daa6f4b5cb1a1fdb3758c0c25(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:16:00,702 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:00,702 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., storeName=cfb45e8932b673e843a8031625dfee57/C, priority=13, startTime=1731975360615; duration=0sec 2024-11-19T00:16:00,702 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:16:00,702 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb45e8932b673e843a8031625dfee57:C 2024-11-19T00:16:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T00:16:00,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:00,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:00,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975420854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:00,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975420855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:00,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975420855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975420857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:00,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975420857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:00,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:00,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975420963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975420963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:00,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975420964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:00,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,966 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975420964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,966 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,966 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:00,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:00,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:00,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975420974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:00,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:00,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:01,063 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb45e8932b673e843a8031625dfee57#A#compaction#498 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:16:01,064 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/af7b6340f3434a7a8aa454da4d62c86c is 175, key is test_row_0/A:col10/1731975360117/Put/seqid=0 2024-11-19T00:16:01,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742419_1595 (size=31515) 2024-11-19T00:16:01,088 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/20b731532e4247948d1d2be50f85bd1e 2024-11-19T00:16:01,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/15ee976c922d4f4699393262d5bc301d is 50, key is test_row_0/B:col10/1731975360212/Put/seqid=0 2024-11-19T00:16:01,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742420_1596 (size=12151) 2024-11-19T00:16:01,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975421173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:01,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975421173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:01,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:01,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:01,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:01,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:01,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:01,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:01,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975421174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:01,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:01,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:01,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:01,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975421174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:01,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:01,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:01,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:01,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975421184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T00:16:01,470 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/af7b6340f3434a7a8aa454da4d62c86c as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/af7b6340f3434a7a8aa454da4d62c86c 2024-11-19T00:16:01,473 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb45e8932b673e843a8031625dfee57/A of cfb45e8932b673e843a8031625dfee57 into af7b6340f3434a7a8aa454da4d62c86c(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:16:01,473 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:01,473 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., storeName=cfb45e8932b673e843a8031625dfee57/A, priority=13, startTime=1731975360615; duration=0sec 2024-11-19T00:16:01,474 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:16:01,474 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb45e8932b673e843a8031625dfee57:A 2024-11-19T00:16:01,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975421483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:01,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:01,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:01,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975421483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:01,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:01,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:01,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:01,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975421484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:01,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:01,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:01,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:01,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:01,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975421484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:01,486 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:01,486 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:01,486 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:01,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975421494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:01,497 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/15ee976c922d4f4699393262d5bc301d 2024-11-19T00:16:01,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:01,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:01,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:01,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/e521383ecb614161a80d76d045eee301 is 50, key is test_row_0/C:col10/1731975360212/Put/seqid=0 2024-11-19T00:16:01,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742421_1597 (size=12151) 2024-11-19T00:16:01,505 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/e521383ecb614161a80d76d045eee301 2024-11-19T00:16:01,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/20b731532e4247948d1d2be50f85bd1e as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/20b731532e4247948d1d2be50f85bd1e 2024-11-19T00:16:01,511 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/20b731532e4247948d1d2be50f85bd1e, entries=150, sequenceid=250, filesize=30.4 K 2024-11-19T00:16:01,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/15ee976c922d4f4699393262d5bc301d as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/15ee976c922d4f4699393262d5bc301d 2024-11-19T00:16:01,514 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/15ee976c922d4f4699393262d5bc301d, entries=150, sequenceid=250, filesize=11.9 K 2024-11-19T00:16:01,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/e521383ecb614161a80d76d045eee301 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/e521383ecb614161a80d76d045eee301 2024-11-19T00:16:01,518 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/e521383ecb614161a80d76d045eee301, entries=150, sequenceid=250, filesize=11.9 K 2024-11-19T00:16:01,518 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for cfb45e8932b673e843a8031625dfee57 in 874ms, sequenceid=250, compaction requested=false 2024-11-19T00:16:01,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:01,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:01,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-19T00:16:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-19T00:16:01,521 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-19T00:16:01,521 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3340 sec 2024-11-19T00:16:01,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.3370 sec 2024-11-19T00:16:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:01,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfb45e8932b673e843a8031625dfee57 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T00:16:01,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=A 2024-11-19T00:16:01,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:01,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=B 2024-11-19T00:16:01,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:01,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=C 2024-11-19T00:16:01,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:02,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e5924839d45f4f479b134f47438a2f53_cfb45e8932b673e843a8031625dfee57 is 50, key is test_row_0/A:col10/1731975360855/Put/seqid=0 2024-11-19T00:16:02,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742422_1598 (size=12404) 2024-11-19T00:16:02,016 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:16:02,020 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e5924839d45f4f479b134f47438a2f53_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e5924839d45f4f479b134f47438a2f53_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:02,021 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/6e2ce54509094c55ae88f0a839bf5264, store: [table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:02,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/6e2ce54509094c55ae88f0a839bf5264 is 175, key is test_row_0/A:col10/1731975360855/Put/seqid=0 2024-11-19T00:16:02,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975422017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:02,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:02,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975422019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:02,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:02,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:02,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975422022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:02,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975422023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:02,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742423_1599 (size=31205) 2024-11-19T00:16:02,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:02,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975422027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:02,033 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,033 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,033 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:02,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:02,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975422134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:02,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:02,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975422134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:02,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:02,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:02,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975422134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:02,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975422134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:02,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975422143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:02,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T00:16:02,334 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:16:02,335 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:16:02,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-19T00:16:02,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T00:16:02,336 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:16:02,337 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:16:02,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:16:02,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:02,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975422343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:02,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:02,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975422343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:02,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:02,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975422343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975422343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:02,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975422354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:02,357 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,357 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,357 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:02,431 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/6e2ce54509094c55ae88f0a839bf5264 2024-11-19T00:16:02,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/f4adcbf15109448d8420c2e11840723f is 50, key is test_row_0/B:col10/1731975360855/Put/seqid=0 2024-11-19T00:16:02,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T00:16:02,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742424_1600 (size=12251) 2024-11-19T00:16:02,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/f4adcbf15109448d8420c2e11840723f 2024-11-19T00:16:02,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/379d2fab38d04f8d8327966f5b8ea3ac is 50, key is test_row_0/C:col10/1731975360855/Put/seqid=0 2024-11-19T00:16:02,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742425_1601 (size=12251) 2024-11-19T00:16:02,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T00:16:02,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:02,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:02,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:02,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:02,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:02,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:02,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T00:16:02,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:02,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:02,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:02,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:02,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:02,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:02,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T00:16:02,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:02,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975422653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:02,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:02,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:02,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975422654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:02,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:02,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:02,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975422654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:02,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:02,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975422654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:02,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:02,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:02,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975422664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:02,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:02,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:02,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T00:16:02,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:02,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:02,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:02,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:02,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:02,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:02,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/379d2fab38d04f8d8327966f5b8ea3ac 2024-11-19T00:16:02,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/6e2ce54509094c55ae88f0a839bf5264 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/6e2ce54509094c55ae88f0a839bf5264 2024-11-19T00:16:02,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/6e2ce54509094c55ae88f0a839bf5264, entries=150, sequenceid=267, filesize=30.5 K 2024-11-19T00:16:02,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/f4adcbf15109448d8420c2e11840723f as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/f4adcbf15109448d8420c2e11840723f 2024-11-19T00:16:02,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/f4adcbf15109448d8420c2e11840723f, entries=150, sequenceid=267, filesize=12.0 K 2024-11-19T00:16:02,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/379d2fab38d04f8d8327966f5b8ea3ac as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/379d2fab38d04f8d8327966f5b8ea3ac 2024-11-19T00:16:02,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/379d2fab38d04f8d8327966f5b8ea3ac, entries=150, sequenceid=267, filesize=12.0 K 2024-11-19T00:16:02,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for cfb45e8932b673e843a8031625dfee57 in 877ms, sequenceid=267, compaction requested=true 2024-11-19T00:16:02,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:02,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb45e8932b673e843a8031625dfee57:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:16:02,873 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:16:02,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:16:02,873 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:16:02,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb45e8932b673e843a8031625dfee57:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:16:02,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:16:02,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb45e8932b673e843a8031625dfee57:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:16:02,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:16:02,874 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93825 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:16:02,874 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): cfb45e8932b673e843a8031625dfee57/A is initiating minor compaction (all files) 2024-11-19T00:16:02,874 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb45e8932b673e843a8031625dfee57/A in TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:02,875 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/af7b6340f3434a7a8aa454da4d62c86c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/20b731532e4247948d1d2be50f85bd1e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/6e2ce54509094c55ae88f0a839bf5264] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp, totalSize=91.6 K 2024-11-19T00:16:02,875 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:02,875 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. files: [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/af7b6340f3434a7a8aa454da4d62c86c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/20b731532e4247948d1d2be50f85bd1e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/6e2ce54509094c55ae88f0a839bf5264] 2024-11-19T00:16:02,875 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36963 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:16:02,875 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): cfb45e8932b673e843a8031625dfee57/B is initiating minor compaction (all files) 2024-11-19T00:16:02,875 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb45e8932b673e843a8031625dfee57/B in TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:02,875 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/26ae2b53980c4689ad02d099da2dc059, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/15ee976c922d4f4699393262d5bc301d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/f4adcbf15109448d8420c2e11840723f] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp, totalSize=36.1 K 2024-11-19T00:16:02,875 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting af7b6340f3434a7a8aa454da4d62c86c, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731975358996 2024-11-19T00:16:02,876 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 26ae2b53980c4689ad02d099da2dc059, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731975358996 2024-11-19T00:16:02,876 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20b731532e4247948d1d2be50f85bd1e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731975360209 2024-11-19T00:16:02,876 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 15ee976c922d4f4699393262d5bc301d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731975360209 2024-11-19T00:16:02,876 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting f4adcbf15109448d8420c2e11840723f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731975360846 2024-11-19T00:16:02,876 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e2ce54509094c55ae88f0a839bf5264, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731975360846 2024-11-19T00:16:02,886 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:02,890 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb45e8932b673e843a8031625dfee57#B#compaction#508 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:16:02,891 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/bc2914d8095e437296608d40206839fd is 50, key is test_row_0/B:col10/1731975360855/Put/seqid=0 2024-11-19T00:16:02,896 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111990798f0d031c4ca69628191c6ceea673_cfb45e8932b673e843a8031625dfee57 store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:02,898 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111990798f0d031c4ca69628191c6ceea673_cfb45e8932b673e843a8031625dfee57, store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:02,899 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111990798f0d031c4ca69628191c6ceea673_cfb45e8932b673e843a8031625dfee57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:02,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742427_1603 (size=4469) 2024-11-19T00:16:02,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742426_1602 (size=12763) 2024-11-19T00:16:02,917 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/bc2914d8095e437296608d40206839fd as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/bc2914d8095e437296608d40206839fd 2024-11-19T00:16:02,922 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb45e8932b673e843a8031625dfee57/B of cfb45e8932b673e843a8031625dfee57 into bc2914d8095e437296608d40206839fd(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:16:02,922 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:02,922 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., storeName=cfb45e8932b673e843a8031625dfee57/B, priority=13, startTime=1731975362873; duration=0sec 2024-11-19T00:16:02,922 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:16:02,922 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb45e8932b673e843a8031625dfee57:B 2024-11-19T00:16:02,922 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:16:02,923 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36963 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:16:02,923 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): cfb45e8932b673e843a8031625dfee57/C is initiating minor compaction (all files) 2024-11-19T00:16:02,923 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb45e8932b673e843a8031625dfee57/C in TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:02,923 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/879f934daa6f4b5cb1a1fdb3758c0c25, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/e521383ecb614161a80d76d045eee301, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/379d2fab38d04f8d8327966f5b8ea3ac] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp, totalSize=36.1 K 2024-11-19T00:16:02,923 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 879f934daa6f4b5cb1a1fdb3758c0c25, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731975358996 2024-11-19T00:16:02,924 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting e521383ecb614161a80d76d045eee301, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731975360209 2024-11-19T00:16:02,924 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 379d2fab38d04f8d8327966f5b8ea3ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731975360846 2024-11-19T00:16:02,929 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb45e8932b673e843a8031625dfee57#C#compaction#509 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:16:02,929 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/b7b3366e21114c399ab385244fdcbaf3 is 50, key is test_row_0/C:col10/1731975360855/Put/seqid=0 2024-11-19T00:16:02,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742428_1604 (size=12763) 2024-11-19T00:16:02,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T00:16:02,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:02,948 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing cfb45e8932b673e843a8031625dfee57 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T00:16:02,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=A 2024-11-19T00:16:02,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:02,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=B 2024-11-19T00:16:02,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:02,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=C 2024-11-19T00:16:02,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:02,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193d8b71214ef1491394db7a8bb6f4eea9_cfb45e8932b673e843a8031625dfee57 is 50, key is test_row_0/A:col10/1731975362020/Put/seqid=0 2024-11-19T00:16:02,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T00:16:02,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742429_1605 (size=12454) 2024-11-19T00:16:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:03,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:03,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:03,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975423176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:03,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:03,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975423176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:03,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:03,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975423177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:03,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:03,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:03,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:03,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975423182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:03,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:03,187 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,187 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975423182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:03,187 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:03,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:03,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975423294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:03,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975423294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:03,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975423294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:03,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975423294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:03,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975423295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:03,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:03,310 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb45e8932b673e843a8031625dfee57#A#compaction#507 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:16:03,310 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/65b573d5b8a849d9bdb1581b2b0e2dd6 is 175, key is test_row_0/A:col10/1731975360855/Put/seqid=0 2024-11-19T00:16:03,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742430_1606 (size=31717) 2024-11-19T00:16:03,325 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/65b573d5b8a849d9bdb1581b2b0e2dd6 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/65b573d5b8a849d9bdb1581b2b0e2dd6 2024-11-19T00:16:03,329 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb45e8932b673e843a8031625dfee57/A of cfb45e8932b673e843a8031625dfee57 into 65b573d5b8a849d9bdb1581b2b0e2dd6(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:16:03,329 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:03,329 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., storeName=cfb45e8932b673e843a8031625dfee57/A, priority=13, startTime=1731975362873; duration=0sec 2024-11-19T00:16:03,329 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:16:03,330 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb45e8932b673e843a8031625dfee57:A 2024-11-19T00:16:03,349 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/b7b3366e21114c399ab385244fdcbaf3 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/b7b3366e21114c399ab385244fdcbaf3 2024-11-19T00:16:03,353 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb45e8932b673e843a8031625dfee57/C of cfb45e8932b673e843a8031625dfee57 into b7b3366e21114c399ab385244fdcbaf3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:16:03,353 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:03,354 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., storeName=cfb45e8932b673e843a8031625dfee57/C, priority=13, startTime=1731975362873; duration=0sec 2024-11-19T00:16:03,354 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:16:03,354 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb45e8932b673e843a8031625dfee57:C 2024-11-19T00:16:03,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:16:03,376 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193d8b71214ef1491394db7a8bb6f4eea9_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193d8b71214ef1491394db7a8bb6f4eea9_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:03,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/4ac24552e6424f2cacb88f29fda255e7, store: [table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:03,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/4ac24552e6424f2cacb88f29fda255e7 is 175, key is test_row_0/A:col10/1731975362020/Put/seqid=0 2024-11-19T00:16:03,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742431_1607 (size=31255) 2024-11-19T00:16:03,384 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/4ac24552e6424f2cacb88f29fda255e7 2024-11-19T00:16:03,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/f4386f513cc4424fb51c76c13e2e7769 is 50, key is test_row_0/B:col10/1731975362020/Put/seqid=0 2024-11-19T00:16:03,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742432_1608 (size=12301) 2024-11-19T00:16:03,436 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/f4386f513cc4424fb51c76c13e2e7769 2024-11-19T00:16:03,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/602baf8fc78d4863b1f3c6c225e65be9 is 50, key is test_row_0/C:col10/1731975362020/Put/seqid=0 2024-11-19T00:16:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T00:16:03,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742433_1609 (size=12301) 2024-11-19T00:16:03,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975423504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:03,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:03,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975423504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:03,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:03,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:03,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975423504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:03,509 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,509 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:03,509 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975423504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:03,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:03,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975423504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:03,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:03,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:03,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975423813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:03,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:03,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975423814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:03,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:03,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:03,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:03,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975423814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:03,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:03,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975423814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:03,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:03,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:03,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975423814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:03,885 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/602baf8fc78d4863b1f3c6c225e65be9 2024-11-19T00:16:03,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/4ac24552e6424f2cacb88f29fda255e7 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/4ac24552e6424f2cacb88f29fda255e7 2024-11-19T00:16:03,895 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/4ac24552e6424f2cacb88f29fda255e7, entries=150, sequenceid=287, filesize=30.5 K 2024-11-19T00:16:03,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/f4386f513cc4424fb51c76c13e2e7769 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/f4386f513cc4424fb51c76c13e2e7769 2024-11-19T00:16:03,904 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/f4386f513cc4424fb51c76c13e2e7769, entries=150, sequenceid=287, filesize=12.0 K 2024-11-19T00:16:03,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/602baf8fc78d4863b1f3c6c225e65be9 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/602baf8fc78d4863b1f3c6c225e65be9 2024-11-19T00:16:03,909 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/602baf8fc78d4863b1f3c6c225e65be9, entries=150, sequenceid=287, filesize=12.0 K 2024-11-19T00:16:03,910 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for cfb45e8932b673e843a8031625dfee57 in 961ms, sequenceid=287, compaction requested=false 2024-11-19T00:16:03,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:03,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:03,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-19T00:16:03,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-19T00:16:03,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-19T00:16:03,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5740 sec 2024-11-19T00:16:03,915 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.5790 sec 2024-11-19T00:16:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:04,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfb45e8932b673e843a8031625dfee57 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T00:16:04,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=A 2024-11-19T00:16:04,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:04,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=B 2024-11-19T00:16:04,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:04,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=C 2024-11-19T00:16:04,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:04,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190217b31bd1c348d5944b8c7bf85d5c7c_cfb45e8932b673e843a8031625dfee57 is 50, key is test_row_0/A:col10/1731975363180/Put/seqid=0 2024-11-19T00:16:04,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742434_1610 (size=14994) 2024-11-19T00:16:04,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:04,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975424347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:04,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:04,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:04,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975424350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:04,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:04,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975424350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:04,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:04,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:04,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975424351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:04,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:04,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975424351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:04,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:04,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975424454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:04,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:04,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:04,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975424464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:04,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:04,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975424464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:04,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:04,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975424464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975424465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T00:16:04,484 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:16:04,486 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:16:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-19T00:16:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T00:16:04,488 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:16:04,489 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:16:04,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:16:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T00:16:04,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T00:16:04,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:04,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:04,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:04,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:04,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:04,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:04,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975424664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:04,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975424676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975424676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:04,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:04,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975424676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:04,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:04,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975424677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:04,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:04,739 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:16:04,742 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190217b31bd1c348d5944b8c7bf85d5c7c_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190217b31bd1c348d5944b8c7bf85d5c7c_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:04,743 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/172aa14176f146d28175cd31234488e4, store: [table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:04,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/172aa14176f146d28175cd31234488e4 is 175, key is test_row_0/A:col10/1731975363180/Put/seqid=0 2024-11-19T00:16:04,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742435_1611 (size=39949) 2024-11-19T00:16:04,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T00:16:04,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:04,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:04,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:04,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:04,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:04,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:04,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T00:16:04,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T00:16:04,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:04,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:04,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:04,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:04,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:04,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:04,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:04,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975424975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:04,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:04,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:04,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975424983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:04,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:04,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975424983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:04,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:04,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975424984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:04,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975424984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:05,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T00:16:05,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:05,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:05,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:05,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:05,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T00:16:05,156 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=305, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/172aa14176f146d28175cd31234488e4 2024-11-19T00:16:05,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/d6d251cd804a4b7c89b7043cb7a55a5c is 50, key is test_row_0/B:col10/1731975363180/Put/seqid=0 2024-11-19T00:16:05,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742436_1612 (size=12301) 2024-11-19T00:16:05,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T00:16:05,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:05,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:05,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:05,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:05,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:05,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T00:16:05,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:05,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:05,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:05,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:05,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:05,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:05,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975425484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:05,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:05,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:05,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:05,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:05,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975425495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975425494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:05,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975425495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:05,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975425495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:05,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:05,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:05,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:05,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T00:16:05,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:05,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:05,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:05,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:05,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:05,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/d6d251cd804a4b7c89b7043cb7a55a5c 2024-11-19T00:16:05,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/c9f29dd8cdf34276ad7166390bbf779a is 50, key is test_row_0/C:col10/1731975363180/Put/seqid=0 2024-11-19T00:16:05,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742437_1613 (size=12301) 2024-11-19T00:16:05,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/c9f29dd8cdf34276ad7166390bbf779a 2024-11-19T00:16:05,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/172aa14176f146d28175cd31234488e4 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/172aa14176f146d28175cd31234488e4 2024-11-19T00:16:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T00:16:05,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/172aa14176f146d28175cd31234488e4, entries=200, sequenceid=305, filesize=39.0 K 2024-11-19T00:16:05,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/d6d251cd804a4b7c89b7043cb7a55a5c as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/d6d251cd804a4b7c89b7043cb7a55a5c 2024-11-19T00:16:05,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/d6d251cd804a4b7c89b7043cb7a55a5c, entries=150, sequenceid=305, filesize=12.0 K 2024-11-19T00:16:05,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/c9f29dd8cdf34276ad7166390bbf779a as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/c9f29dd8cdf34276ad7166390bbf779a 2024-11-19T00:16:05,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/c9f29dd8cdf34276ad7166390bbf779a, entries=150, sequenceid=305, filesize=12.0 K 2024-11-19T00:16:05,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for cfb45e8932b673e843a8031625dfee57 in 1303ms, sequenceid=305, compaction requested=true 2024-11-19T00:16:05,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:05,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb45e8932b673e843a8031625dfee57:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:16:05,631 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:16:05,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:16:05,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb45e8932b673e843a8031625dfee57:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:16:05,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:16:05,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb45e8932b673e843a8031625dfee57:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:16:05,631 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:16:05,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:16:05,632 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102921 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:16:05,632 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): cfb45e8932b673e843a8031625dfee57/A is initiating minor compaction (all files) 2024-11-19T00:16:05,632 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb45e8932b673e843a8031625dfee57/A in TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:05,632 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/65b573d5b8a849d9bdb1581b2b0e2dd6, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/4ac24552e6424f2cacb88f29fda255e7, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/172aa14176f146d28175cd31234488e4] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp, totalSize=100.5 K 2024-11-19T00:16:05,632 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:05,632 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. files: [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/65b573d5b8a849d9bdb1581b2b0e2dd6, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/4ac24552e6424f2cacb88f29fda255e7, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/172aa14176f146d28175cd31234488e4] 2024-11-19T00:16:05,632 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:16:05,632 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): cfb45e8932b673e843a8031625dfee57/B is initiating minor compaction (all files) 2024-11-19T00:16:05,632 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb45e8932b673e843a8031625dfee57/B in TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:05,633 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/bc2914d8095e437296608d40206839fd, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/f4386f513cc4424fb51c76c13e2e7769, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/d6d251cd804a4b7c89b7043cb7a55a5c] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp, totalSize=36.5 K 2024-11-19T00:16:05,633 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65b573d5b8a849d9bdb1581b2b0e2dd6, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731975360846 2024-11-19T00:16:05,633 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ac24552e6424f2cacb88f29fda255e7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731975362020 2024-11-19T00:16:05,633 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting bc2914d8095e437296608d40206839fd, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731975360846 2024-11-19T00:16:05,633 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 172aa14176f146d28175cd31234488e4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731975363180 2024-11-19T00:16:05,633 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting f4386f513cc4424fb51c76c13e2e7769, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731975362020 2024-11-19T00:16:05,633 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting d6d251cd804a4b7c89b7043cb7a55a5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731975363180 2024-11-19T00:16:05,637 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:05,639 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb45e8932b673e843a8031625dfee57#B#compaction#516 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:16:05,639 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/24ef9eb37f5f48b1aafc1694a9590328 is 50, key is test_row_0/B:col10/1731975363180/Put/seqid=0 2024-11-19T00:16:05,640 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119092086922ae144bea9ec7ff68ade035a_cfb45e8932b673e843a8031625dfee57 store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:05,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742438_1614 (size=12915) 2024-11-19T00:16:05,643 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119092086922ae144bea9ec7ff68ade035a_cfb45e8932b673e843a8031625dfee57, store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:05,643 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119092086922ae144bea9ec7ff68ade035a_cfb45e8932b673e843a8031625dfee57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:05,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742439_1615 (size=4469) 2024-11-19T00:16:05,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T00:16:05,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:05,715 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing cfb45e8932b673e843a8031625dfee57 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T00:16:05,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=A 2024-11-19T00:16:05,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:05,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=B 2024-11-19T00:16:05,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:05,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=C 2024-11-19T00:16:05,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:05,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119dee7de1855034098be7ce07f92b96f3e_cfb45e8932b673e843a8031625dfee57 is 50, key is test_row_0/A:col10/1731975364349/Put/seqid=0 2024-11-19T00:16:05,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742440_1616 (size=12454) 2024-11-19T00:16:06,046 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/24ef9eb37f5f48b1aafc1694a9590328 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/24ef9eb37f5f48b1aafc1694a9590328 2024-11-19T00:16:06,049 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb45e8932b673e843a8031625dfee57#A#compaction#517 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:16:06,050 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/eca4f8aeff554c52ab01fb0fe2a22643 is 175, key is test_row_0/A:col10/1731975363180/Put/seqid=0 2024-11-19T00:16:06,053 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb45e8932b673e843a8031625dfee57/B of cfb45e8932b673e843a8031625dfee57 into 24ef9eb37f5f48b1aafc1694a9590328(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:16:06,053 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:06,053 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., storeName=cfb45e8932b673e843a8031625dfee57/B, priority=13, startTime=1731975365631; duration=0sec 2024-11-19T00:16:06,053 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:16:06,054 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb45e8932b673e843a8031625dfee57:B 2024-11-19T00:16:06,054 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:16:06,054 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:16:06,055 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): cfb45e8932b673e843a8031625dfee57/C is initiating minor compaction (all files) 2024-11-19T00:16:06,055 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb45e8932b673e843a8031625dfee57/C in TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:06,055 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/b7b3366e21114c399ab385244fdcbaf3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/602baf8fc78d4863b1f3c6c225e65be9, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/c9f29dd8cdf34276ad7166390bbf779a] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp, totalSize=36.5 K 2024-11-19T00:16:06,055 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting b7b3366e21114c399ab385244fdcbaf3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731975360846 2024-11-19T00:16:06,055 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 602baf8fc78d4863b1f3c6c225e65be9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731975362020 2024-11-19T00:16:06,056 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting c9f29dd8cdf34276ad7166390bbf779a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731975363180 2024-11-19T00:16:06,064 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb45e8932b673e843a8031625dfee57#C#compaction#519 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:16:06,065 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/cdc4aa61191a4f0088b871626faa2a44 is 50, key is test_row_0/C:col10/1731975363180/Put/seqid=0 2024-11-19T00:16:06,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742441_1617 (size=31869) 2024-11-19T00:16:06,080 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/eca4f8aeff554c52ab01fb0fe2a22643 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/eca4f8aeff554c52ab01fb0fe2a22643 2024-11-19T00:16:06,086 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb45e8932b673e843a8031625dfee57/A of cfb45e8932b673e843a8031625dfee57 into eca4f8aeff554c52ab01fb0fe2a22643(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:16:06,086 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:06,086 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., storeName=cfb45e8932b673e843a8031625dfee57/A, priority=13, startTime=1731975365631; duration=0sec 2024-11-19T00:16:06,086 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:16:06,086 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb45e8932b673e843a8031625dfee57:A 2024-11-19T00:16:06,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742442_1618 (size=12915) 2024-11-19T00:16:06,104 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/cdc4aa61191a4f0088b871626faa2a44 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/cdc4aa61191a4f0088b871626faa2a44 2024-11-19T00:16:06,110 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb45e8932b673e843a8031625dfee57/C of cfb45e8932b673e843a8031625dfee57 into cdc4aa61191a4f0088b871626faa2a44(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:16:06,110 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:06,110 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., storeName=cfb45e8932b673e843a8031625dfee57/C, priority=13, startTime=1731975365631; duration=0sec 2024-11-19T00:16:06,110 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:16:06,110 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb45e8932b673e843a8031625dfee57:C 2024-11-19T00:16:06,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:16:06,158 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119dee7de1855034098be7ce07f92b96f3e_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dee7de1855034098be7ce07f92b96f3e_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:06,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/a1c21c0a72a24351b8c3b4124d847160, store: [table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:06,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/a1c21c0a72a24351b8c3b4124d847160 is 175, key is test_row_0/A:col10/1731975364349/Put/seqid=0 2024-11-19T00:16:06,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742443_1619 (size=31255) 2024-11-19T00:16:06,182 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/a1c21c0a72a24351b8c3b4124d847160 2024-11-19T00:16:06,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/215a2fabf91e43359c971bf8f0ba2238 is 50, key is test_row_0/B:col10/1731975364349/Put/seqid=0 2024-11-19T00:16:06,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742444_1620 (size=12301) 2024-11-19T00:16:06,255 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/215a2fabf91e43359c971bf8f0ba2238 2024-11-19T00:16:06,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/5c32fd5630d14252b790e98cefe3f962 is 50, key is test_row_0/C:col10/1731975364349/Put/seqid=0 2024-11-19T00:16:06,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742445_1621 (size=12301) 2024-11-19T00:16:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:06,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:06,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975426515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:06,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:06,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975426517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:06,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:06,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975426518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:06,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:06,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975426519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:06,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:06,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975426520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:06,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:06,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975426624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:06,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975426624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T00:16:06,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975426634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:06,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:06,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975426634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:06,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:06,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:06,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975426634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:06,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:06,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:06,700 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/5c32fd5630d14252b790e98cefe3f962 2024-11-19T00:16:06,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/a1c21c0a72a24351b8c3b4124d847160 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/a1c21c0a72a24351b8c3b4124d847160 2024-11-19T00:16:06,705 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/a1c21c0a72a24351b8c3b4124d847160, entries=150, sequenceid=321, filesize=30.5 K 2024-11-19T00:16:06,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/215a2fabf91e43359c971bf8f0ba2238 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/215a2fabf91e43359c971bf8f0ba2238 2024-11-19T00:16:06,708 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/215a2fabf91e43359c971bf8f0ba2238, entries=150, sequenceid=321, filesize=12.0 K 2024-11-19T00:16:06,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/5c32fd5630d14252b790e98cefe3f962 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/5c32fd5630d14252b790e98cefe3f962 2024-11-19T00:16:06,712 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/5c32fd5630d14252b790e98cefe3f962, entries=150, sequenceid=321, filesize=12.0 K 2024-11-19T00:16:06,713 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for cfb45e8932b673e843a8031625dfee57 in 998ms, sequenceid=321, compaction requested=false 2024-11-19T00:16:06,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:06,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:06,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-19T00:16:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-19T00:16:06,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-19T00:16:06,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2250 sec 2024-11-19T00:16:06,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.2330 sec 2024-11-19T00:16:06,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:06,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfb45e8932b673e843a8031625dfee57 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T00:16:06,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=A 2024-11-19T00:16:06,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:06,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=B 2024-11-19T00:16:06,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:06,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=C 2024-11-19T00:16:06,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:06,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198fd1805e34cd4c2ab5b0089beb16673c_cfb45e8932b673e843a8031625dfee57 is 50, key is test_row_0/A:col10/1731975366510/Put/seqid=0 2024-11-19T00:16:06,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742446_1622 (size=14994) 2024-11-19T00:16:06,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:06,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:06,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975426852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:06,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975426851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:06,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:06,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975426853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:06,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:06,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975426856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:06,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:06,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:06,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975426858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:06,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:06,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975426964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:06,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:06,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975426964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:06,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:06,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975426964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:06,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:06,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975426964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:06,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:06,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975426973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:06,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:06,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:07,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975427174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:07,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975427174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:07,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975427174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:07,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975427174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:07,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975427184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:07,249 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:16:07,254 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198fd1805e34cd4c2ab5b0089beb16673c_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198fd1805e34cd4c2ab5b0089beb16673c_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:07,255 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/c252ac79bf874cb8b4b444067a907927, store: [table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:07,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/c252ac79bf874cb8b4b444067a907927 is 175, key is test_row_0/A:col10/1731975366510/Put/seqid=0 2024-11-19T00:16:07,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742447_1623 (size=39949) 2024-11-19T00:16:07,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975427484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:07,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:07,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975427484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:07,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:07,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975427484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:07,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:07,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975427484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:07,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:07,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975427493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:07,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:07,660 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/c252ac79bf874cb8b4b444067a907927 2024-11-19T00:16:07,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/c8b053b28fad42d780eb6201797e33c1 is 50, key is test_row_0/B:col10/1731975366510/Put/seqid=0 2024-11-19T00:16:07,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742448_1624 (size=12301) 2024-11-19T00:16:07,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/c8b053b28fad42d780eb6201797e33c1 2024-11-19T00:16:07,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/afcba2ddc6fd4d028e59faabd22a7c07 is 50, key is test_row_0/C:col10/1731975366510/Put/seqid=0 2024-11-19T00:16:07,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742449_1625 (size=12301) 2024-11-19T00:16:07,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/afcba2ddc6fd4d028e59faabd22a7c07 2024-11-19T00:16:07,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/c252ac79bf874cb8b4b444067a907927 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/c252ac79bf874cb8b4b444067a907927 2024-11-19T00:16:07,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/c252ac79bf874cb8b4b444067a907927, entries=200, sequenceid=342, filesize=39.0 K 2024-11-19T00:16:07,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/c8b053b28fad42d780eb6201797e33c1 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/c8b053b28fad42d780eb6201797e33c1 2024-11-19T00:16:07,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/c8b053b28fad42d780eb6201797e33c1, entries=150, sequenceid=342, filesize=12.0 K 2024-11-19T00:16:07,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/afcba2ddc6fd4d028e59faabd22a7c07 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/afcba2ddc6fd4d028e59faabd22a7c07 2024-11-19T00:16:07,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/afcba2ddc6fd4d028e59faabd22a7c07, entries=150, sequenceid=342, filesize=12.0 K 2024-11-19T00:16:07,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for cfb45e8932b673e843a8031625dfee57 in 954ms, sequenceid=342, compaction requested=true 2024-11-19T00:16:07,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:07,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb45e8932b673e843a8031625dfee57:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:16:07,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:16:07,791 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:16:07,791 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:16:07,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb45e8932b673e843a8031625dfee57:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:16:07,792 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:16:07,792 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): cfb45e8932b673e843a8031625dfee57/A is initiating minor compaction (all files) 2024-11-19T00:16:07,792 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb45e8932b673e843a8031625dfee57/A in TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:07,792 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/eca4f8aeff554c52ab01fb0fe2a22643, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/a1c21c0a72a24351b8c3b4124d847160, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/c252ac79bf874cb8b4b444067a907927] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp, totalSize=100.7 K 2024-11-19T00:16:07,792 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:07,792 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. files: [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/eca4f8aeff554c52ab01fb0fe2a22643, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/a1c21c0a72a24351b8c3b4124d847160, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/c252ac79bf874cb8b4b444067a907927] 2024-11-19T00:16:07,793 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:16:07,793 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): cfb45e8932b673e843a8031625dfee57/B is initiating minor compaction (all files) 2024-11-19T00:16:07,793 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb45e8932b673e843a8031625dfee57/B in TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:07,793 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/24ef9eb37f5f48b1aafc1694a9590328, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/215a2fabf91e43359c971bf8f0ba2238, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/c8b053b28fad42d780eb6201797e33c1] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp, totalSize=36.6 K 2024-11-19T00:16:07,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:16:07,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb45e8932b673e843a8031625dfee57:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:16:07,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:16:07,793 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting eca4f8aeff554c52ab01fb0fe2a22643, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731975363180 2024-11-19T00:16:07,794 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1c21c0a72a24351b8c3b4124d847160, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731975364348 2024-11-19T00:16:07,794 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 24ef9eb37f5f48b1aafc1694a9590328, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731975363180 2024-11-19T00:16:07,794 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting c252ac79bf874cb8b4b444067a907927, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731975366510 2024-11-19T00:16:07,794 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 215a2fabf91e43359c971bf8f0ba2238, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731975364348 2024-11-19T00:16:07,794 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting c8b053b28fad42d780eb6201797e33c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731975366510 2024-11-19T00:16:07,817 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb45e8932b673e843a8031625dfee57#B#compaction#525 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:16:07,818 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/59da562195f64d05a850495d7e0eb22e is 50, key is test_row_0/B:col10/1731975366510/Put/seqid=0 2024-11-19T00:16:07,822 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:07,838 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111956040514d8e9423682931e1f9db602f1_cfb45e8932b673e843a8031625dfee57 store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:07,840 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111956040514d8e9423682931e1f9db602f1_cfb45e8932b673e843a8031625dfee57, store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:07,840 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111956040514d8e9423682931e1f9db602f1_cfb45e8932b673e843a8031625dfee57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:07,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742450_1626 (size=13017) 2024-11-19T00:16:07,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742451_1627 (size=4469) 2024-11-19T00:16:07,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:07,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfb45e8932b673e843a8031625dfee57 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T00:16:07,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=A 2024-11-19T00:16:07,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:07,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=B 2024-11-19T00:16:07,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:07,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=C 2024-11-19T00:16:07,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:08,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111923cd5c4cf2df4f7eacb19e91f2d3a477_cfb45e8932b673e843a8031625dfee57 is 50, key is test_row_0/A:col10/1731975367995/Put/seqid=0 2024-11-19T00:16:08,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742452_1628 (size=12454) 2024-11-19T00:16:08,017 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:16:08,019 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111923cd5c4cf2df4f7eacb19e91f2d3a477_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111923cd5c4cf2df4f7eacb19e91f2d3a477_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:08,020 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/f1cd6a485b434703ab069236499ce6e5, store: [table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:08,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/f1cd6a485b434703ab069236499ce6e5 is 175, key is test_row_0/A:col10/1731975367995/Put/seqid=0 2024-11-19T00:16:08,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:08,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975428014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:08,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:08,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:08,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975428015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:08,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:08,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:08,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975428017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:08,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:08,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742453_1629 (size=31255) 2024-11-19T00:16:08,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:08,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975428022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:08,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:08,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975428023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:08,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:08,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:08,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975428124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:08,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:08,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:08,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975428134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:08,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:08,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:08,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975428134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:08,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:08,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975428134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:08,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:08,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:08,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:08,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975428134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:08,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:08,257 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/59da562195f64d05a850495d7e0eb22e as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/59da562195f64d05a850495d7e0eb22e 2024-11-19T00:16:08,261 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb45e8932b673e843a8031625dfee57/B of cfb45e8932b673e843a8031625dfee57 into 59da562195f64d05a850495d7e0eb22e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:16:08,261 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:08,261 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., storeName=cfb45e8932b673e843a8031625dfee57/B, priority=13, startTime=1731975367791; duration=0sec 2024-11-19T00:16:08,261 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:16:08,261 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb45e8932b673e843a8031625dfee57:B 2024-11-19T00:16:08,261 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T00:16:08,262 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T00:16:08,262 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): cfb45e8932b673e843a8031625dfee57/C is initiating minor compaction (all files) 2024-11-19T00:16:08,262 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb45e8932b673e843a8031625dfee57/C in TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:08,263 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/cdc4aa61191a4f0088b871626faa2a44, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/5c32fd5630d14252b790e98cefe3f962, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/afcba2ddc6fd4d028e59faabd22a7c07] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp, totalSize=36.6 K 2024-11-19T00:16:08,263 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting cdc4aa61191a4f0088b871626faa2a44, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731975363180 2024-11-19T00:16:08,263 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c32fd5630d14252b790e98cefe3f962, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731975364348 2024-11-19T00:16:08,263 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting afcba2ddc6fd4d028e59faabd22a7c07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731975366510 2024-11-19T00:16:08,271 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb45e8932b673e843a8031625dfee57#A#compaction#526 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:16:08,271 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/587f962bfce34088b3eb7dec58781a7c is 175, key is test_row_0/A:col10/1731975366510/Put/seqid=0 2024-11-19T00:16:08,275 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb45e8932b673e843a8031625dfee57#C#compaction#528 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:16:08,276 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/d936f29648df473a9280bc7f8eb0e6b9 is 50, key is test_row_0/C:col10/1731975366510/Put/seqid=0 2024-11-19T00:16:08,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742454_1630 (size=31971) 2024-11-19T00:16:08,282 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/587f962bfce34088b3eb7dec58781a7c as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/587f962bfce34088b3eb7dec58781a7c 2024-11-19T00:16:08,288 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb45e8932b673e843a8031625dfee57/A of cfb45e8932b673e843a8031625dfee57 into 587f962bfce34088b3eb7dec58781a7c(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:16:08,288 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:08,288 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., storeName=cfb45e8932b673e843a8031625dfee57/A, priority=13, startTime=1731975367791; duration=0sec 2024-11-19T00:16:08,288 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:16:08,288 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb45e8932b673e843a8031625dfee57:A 2024-11-19T00:16:08,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742455_1631 (size=13017) 2024-11-19T00:16:08,302 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/d936f29648df473a9280bc7f8eb0e6b9 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/d936f29648df473a9280bc7f8eb0e6b9 2024-11-19T00:16:08,307 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb45e8932b673e843a8031625dfee57/C of cfb45e8932b673e843a8031625dfee57 into d936f29648df473a9280bc7f8eb0e6b9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:16:08,307 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:08,307 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., storeName=cfb45e8932b673e843a8031625dfee57/C, priority=13, startTime=1731975367793; duration=0sec 2024-11-19T00:16:08,307 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:16:08,307 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb45e8932b673e843a8031625dfee57:C 2024-11-19T00:16:08,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:08,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975428334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:08,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:08,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975428344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975428344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:08,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:08,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975428344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:08,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:08,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975428344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:08,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:08,425 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=358, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/f1cd6a485b434703ab069236499ce6e5 2024-11-19T00:16:08,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/364794a9e65b49efa8d1509396a7bfd0 is 50, key is test_row_0/B:col10/1731975367995/Put/seqid=0 2024-11-19T00:16:08,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742456_1632 (size=12301) 2024-11-19T00:16:08,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/364794a9e65b49efa8d1509396a7bfd0 2024-11-19T00:16:08,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/79bd722d9e9e4ebc92249673a897e9d3 is 50, key is test_row_0/C:col10/1731975367995/Put/seqid=0 2024-11-19T00:16:08,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742457_1633 (size=12301) 2024-11-19T00:16:08,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T00:16:08,644 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:16:08,645 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:16:08,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-19T00:16:08,647 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:16:08,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T00:16:08,647 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:16:08,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:16:08,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:08,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975428644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:08,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:08,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:08,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975428653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:08,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:08,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:08,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975428654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:08,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975428654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:08,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975428654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:08,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T00:16:08,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T00:16:08,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:08,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:08,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:08,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:08,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:08,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:08,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/79bd722d9e9e4ebc92249673a897e9d3 2024-11-19T00:16:08,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/f1cd6a485b434703ab069236499ce6e5 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/f1cd6a485b434703ab069236499ce6e5 2024-11-19T00:16:08,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/f1cd6a485b434703ab069236499ce6e5, entries=150, sequenceid=358, filesize=30.5 K 2024-11-19T00:16:08,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/364794a9e65b49efa8d1509396a7bfd0 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/364794a9e65b49efa8d1509396a7bfd0 2024-11-19T00:16:08,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/364794a9e65b49efa8d1509396a7bfd0, entries=150, sequenceid=358, filesize=12.0 K 2024-11-19T00:16:08,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/79bd722d9e9e4ebc92249673a897e9d3 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/79bd722d9e9e4ebc92249673a897e9d3 2024-11-19T00:16:08,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/79bd722d9e9e4ebc92249673a897e9d3, entries=150, sequenceid=358, filesize=12.0 K 2024-11-19T00:16:08,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for cfb45e8932b673e843a8031625dfee57 in 952ms, sequenceid=358, compaction requested=false 2024-11-19T00:16:08,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:08,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T00:16:08,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:08,955 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing cfb45e8932b673e843a8031625dfee57 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T00:16:08,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=A 2024-11-19T00:16:08,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:08,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=B 2024-11-19T00:16:08,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:08,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=C 2024-11-19T00:16:08,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T00:16:08,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d2387017a8bd4e6297801596781a1f5b_cfb45e8932b673e843a8031625dfee57 is 50, key is test_row_0/A:col10/1731975368015/Put/seqid=0 2024-11-19T00:16:08,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742458_1634 (size=12454) 2024-11-19T00:16:08,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:16:08,985 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d2387017a8bd4e6297801596781a1f5b_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d2387017a8bd4e6297801596781a1f5b_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:08,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/f82fbabdb0134a828c9159d847fd561d, store: [table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:08,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/f82fbabdb0134a828c9159d847fd561d is 175, key is test_row_0/A:col10/1731975368015/Put/seqid=0 2024-11-19T00:16:09,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742459_1635 (size=31255) 2024-11-19T00:16:09,009 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=378, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/f82fbabdb0134a828c9159d847fd561d 2024-11-19T00:16:09,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/d2087119c5e845d1bef9d84b62e84be4 is 50, key is test_row_0/B:col10/1731975368015/Put/seqid=0 2024-11-19T00:16:09,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742460_1636 (size=12301) 2024-11-19T00:16:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:09,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:09,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975429177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:09,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975429178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:09,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:09,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:09,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975429221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975429221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:09,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975429221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T00:16:09,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975429324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:09,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975429324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:09,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975429334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:09,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975429334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:09,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975429337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:09,436 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/d2087119c5e845d1bef9d84b62e84be4 2024-11-19T00:16:09,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/be02722038c347f782c799a6ceb4bac3 is 50, key is test_row_0/C:col10/1731975368015/Put/seqid=0 2024-11-19T00:16:09,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742461_1637 (size=12301) 2024-11-19T00:16:09,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975429534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:09,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:09,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975429534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:09,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:09,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:09,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975429544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975429544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:09,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:09,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975429545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:09,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:09,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:09,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:09,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T00:16:09,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975429844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:09,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975429844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:09,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:09,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975429853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:09,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975429853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:09,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975429854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:09,864 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/be02722038c347f782c799a6ceb4bac3 2024-11-19T00:16:09,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/f82fbabdb0134a828c9159d847fd561d as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/f82fbabdb0134a828c9159d847fd561d 2024-11-19T00:16:09,870 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/f82fbabdb0134a828c9159d847fd561d, entries=150, sequenceid=378, filesize=30.5 K 2024-11-19T00:16:09,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/d2087119c5e845d1bef9d84b62e84be4 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/d2087119c5e845d1bef9d84b62e84be4 2024-11-19T00:16:09,873 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/d2087119c5e845d1bef9d84b62e84be4, entries=150, sequenceid=378, filesize=12.0 K 2024-11-19T00:16:09,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/be02722038c347f782c799a6ceb4bac3 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/be02722038c347f782c799a6ceb4bac3 2024-11-19T00:16:09,876 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/be02722038c347f782c799a6ceb4bac3, entries=150, sequenceid=378, filesize=12.0 K 2024-11-19T00:16:09,877 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for cfb45e8932b673e843a8031625dfee57 in 923ms, sequenceid=378, compaction requested=true 2024-11-19T00:16:09,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:09,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:09,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-19T00:16:09,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-19T00:16:09,879 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-19T00:16:09,879 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2310 sec 2024-11-19T00:16:09,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.2350 sec 2024-11-19T00:16:10,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:10,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfb45e8932b673e843a8031625dfee57 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T00:16:10,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=A 2024-11-19T00:16:10,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:10,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=B 2024-11-19T00:16:10,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:10,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=C 2024-11-19T00:16:10,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:10,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ac52e0a34cc04eaaa0c94dfaf9b1206d_cfb45e8932b673e843a8031625dfee57 is 50, key is test_row_0/A:col10/1731975369178/Put/seqid=0 2024-11-19T00:16:10,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742462_1638 (size=14994) 2024-11-19T00:16:10,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:10,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975430377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:10,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:10,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975430377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:10,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:10,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975430379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:10,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:10,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975430385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:10,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975430386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:10,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975430493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975430493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:10,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975430493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:10,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975430494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975430494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:10,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:10,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975430704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:10,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:10,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:10,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975430704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:10,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:10,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:10,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975430704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:10,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:10,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975430704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:10,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:10,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975430704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:10,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:10,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:10,778 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:16:10,781 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ac52e0a34cc04eaaa0c94dfaf9b1206d_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ac52e0a34cc04eaaa0c94dfaf9b1206d_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:10,782 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/a5a657640ec34e6b99959a7ab7b5dd11, store: [table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:10,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/a5a657640ec34e6b99959a7ab7b5dd11 is 175, key is test_row_0/A:col10/1731975369178/Put/seqid=0 2024-11-19T00:16:10,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742463_1639 (size=39949) 2024-11-19T00:16:10,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T00:16:10,804 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:16:10,805 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T00:16:10,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-19T00:16:10,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T00:16:10,806 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T00:16:10,807 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T00:16:10,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T00:16:10,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T00:16:10,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T00:16:10,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:10,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:10,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:10,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:10,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:10,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:11,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975431014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:11,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975431014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:11,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975431014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:11,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:11,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:11,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975431015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:11,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:11,016 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:11,016 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:11,016 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:11,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975431015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:11,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:11,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:11,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:11,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T00:16:11,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:11,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:11,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:11,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:11,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T00:16:11,187 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=396, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/a5a657640ec34e6b99959a7ab7b5dd11 2024-11-19T00:16:11,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/1385757cd20344829e48c9cdd7bc4c7b is 50, key is test_row_0/B:col10/1731975369178/Put/seqid=0 2024-11-19T00:16:11,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742464_1640 (size=12301) 2024-11-19T00:16:11,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T00:16:11,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:11,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:11,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:11,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:11,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:11,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T00:16:11,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:11,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:11,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:11,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:11,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:11,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T00:16:11,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34742 deadline: 1731975431523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:11,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:11,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:11,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34720 deadline: 1731975431523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:11,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:11,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:11,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:11,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34734 deadline: 1731975431523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:11,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:11,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34782 deadline: 1731975431524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:11,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:11,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T00:16:11,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:11,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:11,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:11,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:11,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38567 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34788 deadline: 1731975431524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 2024-11-19T00:16:11,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:11,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 , the old value is region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:11,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T00:16:11,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., hostname=6b1531871d35,38567,1731975194317, seqNum=5 because the exception is null or not the one we care about 2024-11-19T00:16:11,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T00:16:11,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:11,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:11,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:11,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:11,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:11,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:11,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/1385757cd20344829e48c9cdd7bc4c7b 2024-11-19T00:16:11,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/40241a3218134de793e7dee2fa01453d is 50, key is test_row_0/C:col10/1731975369178/Put/seqid=0 2024-11-19T00:16:11,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742465_1641 (size=12301) 2024-11-19T00:16:11,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T00:16:11,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:11,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:11,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:11,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:11,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:11,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:11,734 INFO [Thread-2429 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2429. 2024-11-19T00:16:11,734 DEBUG [Thread-2429 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:16:11,734 DEBUG [Thread-2429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:11,734 DEBUG [Thread-2429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:11,735 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:16:11,735 INFO [Thread-2423 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2423. 2024-11-19T00:16:11,735 DEBUG [Thread-2423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:16:11,735 DEBUG [Thread-2423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:11,736 DEBUG [Thread-2423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:11,736 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:16:11,736 INFO [Thread-2427 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2427. 2024-11-19T00:16:11,736 DEBUG [Thread-2427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:16:11,736 DEBUG [Thread-2427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:11,736 DEBUG [Thread-2427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:11,736 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:16:11,738 INFO [Thread-2431 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2431. 2024-11-19T00:16:11,738 DEBUG [Thread-2431 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:16:11,738 DEBUG [Thread-2431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:11,738 DEBUG [Thread-2431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:11,738 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:16:11,739 INFO [Thread-2425 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2425. 2024-11-19T00:16:11,739 DEBUG [Thread-2425 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:16:11,739 DEBUG [Thread-2425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:11,739 DEBUG [Thread-2425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:11,739 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:16:11,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T00:16:11,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:11,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:11,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:11,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:11,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T00:16:12,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T00:16:12,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:12,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:12,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:12,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:12,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:12,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T00:16:12,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/40241a3218134de793e7dee2fa01453d 2024-11-19T00:16:12,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/a5a657640ec34e6b99959a7ab7b5dd11 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/a5a657640ec34e6b99959a7ab7b5dd11 2024-11-19T00:16:12,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/a5a657640ec34e6b99959a7ab7b5dd11, entries=200, sequenceid=396, filesize=39.0 K 2024-11-19T00:16:12,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/1385757cd20344829e48c9cdd7bc4c7b as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/1385757cd20344829e48c9cdd7bc4c7b 2024-11-19T00:16:12,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/1385757cd20344829e48c9cdd7bc4c7b, entries=150, sequenceid=396, filesize=12.0 K 2024-11-19T00:16:12,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/40241a3218134de793e7dee2fa01453d as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/40241a3218134de793e7dee2fa01453d 2024-11-19T00:16:12,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/40241a3218134de793e7dee2fa01453d, entries=150, sequenceid=396, filesize=12.0 K 2024-11-19T00:16:12,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for cfb45e8932b673e843a8031625dfee57 in 1697ms, sequenceid=396, compaction requested=true 2024-11-19T00:16:12,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:12,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb45e8932b673e843a8031625dfee57:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T00:16:12,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:16:12,056 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:16:12,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb45e8932b673e843a8031625dfee57:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T00:16:12,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:16:12,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb45e8932b673e843a8031625dfee57:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T00:16:12,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:16:12,056 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:16:12,057 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134430 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:16:12,057 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:16:12,057 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1541): cfb45e8932b673e843a8031625dfee57/A is initiating minor compaction (all files) 2024-11-19T00:16:12,057 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): cfb45e8932b673e843a8031625dfee57/B is initiating minor compaction (all files) 2024-11-19T00:16:12,057 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb45e8932b673e843a8031625dfee57/A in TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:12,057 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb45e8932b673e843a8031625dfee57/B in TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:12,057 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/587f962bfce34088b3eb7dec58781a7c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/f1cd6a485b434703ab069236499ce6e5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/f82fbabdb0134a828c9159d847fd561d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/a5a657640ec34e6b99959a7ab7b5dd11] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp, totalSize=131.3 K 2024-11-19T00:16:12,057 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/59da562195f64d05a850495d7e0eb22e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/364794a9e65b49efa8d1509396a7bfd0, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/d2087119c5e845d1bef9d84b62e84be4, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/1385757cd20344829e48c9cdd7bc4c7b] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp, totalSize=48.8 K 2024-11-19T00:16:12,058 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:12,058 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. files: [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/587f962bfce34088b3eb7dec58781a7c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/f1cd6a485b434703ab069236499ce6e5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/f82fbabdb0134a828c9159d847fd561d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/a5a657640ec34e6b99959a7ab7b5dd11] 2024-11-19T00:16:12,058 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59da562195f64d05a850495d7e0eb22e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731975366510 2024-11-19T00:16:12,058 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting 587f962bfce34088b3eb7dec58781a7c, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731975366510 2024-11-19T00:16:12,058 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 364794a9e65b49efa8d1509396a7bfd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731975366855 2024-11-19T00:16:12,058 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting f1cd6a485b434703ab069236499ce6e5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731975366855 2024-11-19T00:16:12,058 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2087119c5e845d1bef9d84b62e84be4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731975368014 2024-11-19T00:16:12,058 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting f82fbabdb0134a828c9159d847fd561d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731975368014 2024-11-19T00:16:12,058 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1385757cd20344829e48c9cdd7bc4c7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731975369178 2024-11-19T00:16:12,058 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] compactions.Compactor(225): Compacting a5a657640ec34e6b99959a7ab7b5dd11, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731975369178 2024-11-19T00:16:12,063 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:12,064 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb45e8932b673e843a8031625dfee57#B#compaction#537 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:16:12,064 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411195b8f7eb77db649b2acfbbae5e5676588_cfb45e8932b673e843a8031625dfee57 store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:12,064 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/e07d94477aaf4029a545cae31a00203b is 50, key is test_row_0/B:col10/1731975369178/Put/seqid=0 2024-11-19T00:16:12,067 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411195b8f7eb77db649b2acfbbae5e5676588_cfb45e8932b673e843a8031625dfee57, store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:12,067 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195b8f7eb77db649b2acfbbae5e5676588_cfb45e8932b673e843a8031625dfee57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:12,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742466_1642 (size=13153) 2024-11-19T00:16:12,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742467_1643 (size=4469) 2024-11-19T00:16:12,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38567 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T00:16:12,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:12,184 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing cfb45e8932b673e843a8031625dfee57 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T00:16:12,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=A 2024-11-19T00:16:12,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:12,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=B 2024-11-19T00:16:12,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:12,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=C 2024-11-19T00:16:12,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:12,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192aba74633d6843658aafa8494ca2be2b_cfb45e8932b673e843a8031625dfee57 is 50, key is test_row_0/A:col10/1731975370383/Put/seqid=0 2024-11-19T00:16:12,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742468_1644 (size=12454) 2024-11-19T00:16:12,472 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb45e8932b673e843a8031625dfee57#A#compaction#538 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:16:12,473 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/e78e9b0675854607b192d50c753fe080 is 175, key is test_row_0/A:col10/1731975369178/Put/seqid=0 2024-11-19T00:16:12,477 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/e07d94477aaf4029a545cae31a00203b as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/e07d94477aaf4029a545cae31a00203b 2024-11-19T00:16:12,482 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfb45e8932b673e843a8031625dfee57/B of cfb45e8932b673e843a8031625dfee57 into e07d94477aaf4029a545cae31a00203b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:16:12,482 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:12,482 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., storeName=cfb45e8932b673e843a8031625dfee57/B, priority=12, startTime=1731975372056; duration=0sec 2024-11-19T00:16:12,482 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T00:16:12,483 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb45e8932b673e843a8031625dfee57:B 2024-11-19T00:16:12,483 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T00:16:12,484 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T00:16:12,484 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1541): cfb45e8932b673e843a8031625dfee57/C is initiating minor compaction (all files) 2024-11-19T00:16:12,484 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb45e8932b673e843a8031625dfee57/C in TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:12,484 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/d936f29648df473a9280bc7f8eb0e6b9, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/79bd722d9e9e4ebc92249673a897e9d3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/be02722038c347f782c799a6ceb4bac3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/40241a3218134de793e7dee2fa01453d] into tmpdir=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp, totalSize=48.8 K 2024-11-19T00:16:12,485 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting d936f29648df473a9280bc7f8eb0e6b9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731975366510 2024-11-19T00:16:12,485 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79bd722d9e9e4ebc92249673a897e9d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731975366855 2024-11-19T00:16:12,485 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting be02722038c347f782c799a6ceb4bac3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731975368014 2024-11-19T00:16:12,485 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40241a3218134de793e7dee2fa01453d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731975369178 2024-11-19T00:16:12,494 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb45e8932b673e843a8031625dfee57#C#compaction#540 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T00:16:12,494 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/1a9c1cf7496440e5bb1f58b709d9fb8d is 50, key is test_row_0/C:col10/1731975369178/Put/seqid=0 2024-11-19T00:16:12,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742469_1645 (size=32107) 2024-11-19T00:16:12,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742470_1646 (size=13153) 2024-11-19T00:16:12,506 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/1a9c1cf7496440e5bb1f58b709d9fb8d as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/1a9c1cf7496440e5bb1f58b709d9fb8d 2024-11-19T00:16:12,506 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/e78e9b0675854607b192d50c753fe080 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/e78e9b0675854607b192d50c753fe080 2024-11-19T00:16:12,509 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfb45e8932b673e843a8031625dfee57/C of cfb45e8932b673e843a8031625dfee57 into 1a9c1cf7496440e5bb1f58b709d9fb8d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:16:12,509 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfb45e8932b673e843a8031625dfee57/A of cfb45e8932b673e843a8031625dfee57 into e78e9b0675854607b192d50c753fe080(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T00:16:12,509 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:12,509 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:12,509 INFO [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., storeName=cfb45e8932b673e843a8031625dfee57/A, priority=12, startTime=1731975372056; duration=0sec 2024-11-19T00:16:12,509 INFO [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., storeName=cfb45e8932b673e843a8031625dfee57/C, priority=12, startTime=1731975372056; duration=0sec 2024-11-19T00:16:12,509 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:16:12,509 DEBUG [RS:0;6b1531871d35:38567-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb45e8932b673e843a8031625dfee57:A 2024-11-19T00:16:12,509 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T00:16:12,510 DEBUG [RS:0;6b1531871d35:38567-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb45e8932b673e843a8031625dfee57:C 2024-11-19T00:16:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38567 {}] regionserver.HRegion(8855): Flush requested on cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:12,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. as already flushing 2024-11-19T00:16:12,536 INFO [Thread-2414 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2414. 2024-11-19T00:16:12,536 INFO [Thread-2420 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2420. 2024-11-19T00:16:12,536 INFO [Thread-2418 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2418. 2024-11-19T00:16:12,536 DEBUG [Thread-2414 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:16:12,536 DEBUG [Thread-2420 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:16:12,536 DEBUG [Thread-2418 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:16:12,536 DEBUG [Thread-2414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:12,536 DEBUG [Thread-2420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:12,536 DEBUG [Thread-2418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:12,536 DEBUG [Thread-2420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:12,536 DEBUG [Thread-2414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:12,536 DEBUG [Thread-2418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:12,536 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:16:12,536 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:16:12,536 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:16:12,537 INFO [Thread-2416 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2416. 2024-11-19T00:16:12,537 INFO [Thread-2412 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2412. 2024-11-19T00:16:12,537 DEBUG [Thread-2416 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:16:12,537 DEBUG [Thread-2412 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T00:16:12,538 DEBUG [Thread-2416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:12,538 DEBUG [Thread-2412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:12,538 DEBUG [Thread-2412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:12,538 DEBUG [Thread-2416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:12,538 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:16:12,540 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:16:12,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:16:12,596 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192aba74633d6843658aafa8494ca2be2b_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192aba74633d6843658aafa8494ca2be2b_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:12,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/8f15a3092c564cd7b3143016fc3257a7, store: [table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:12,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/8f15a3092c564cd7b3143016fc3257a7 is 175, key is test_row_0/A:col10/1731975370383/Put/seqid=0 2024-11-19T00:16:12,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742471_1647 (size=31255) 2024-11-19T00:16:12,653 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T00:16:12,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T00:16:13,003 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=411, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/8f15a3092c564cd7b3143016fc3257a7 2024-11-19T00:16:13,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/6b3cdad23bf8477bbb101e36149be251 is 50, key is test_row_0/B:col10/1731975370383/Put/seqid=0 2024-11-19T00:16:13,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742472_1648 (size=12301) 2024-11-19T00:16:13,415 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/6b3cdad23bf8477bbb101e36149be251 2024-11-19T00:16:13,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/898ac263ede64076893d7174b1b6604b is 50, key is test_row_0/C:col10/1731975370383/Put/seqid=0 2024-11-19T00:16:13,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742473_1649 (size=12301) 2024-11-19T00:16:13,825 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/898ac263ede64076893d7174b1b6604b 2024-11-19T00:16:13,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/8f15a3092c564cd7b3143016fc3257a7 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/8f15a3092c564cd7b3143016fc3257a7 2024-11-19T00:16:13,832 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/8f15a3092c564cd7b3143016fc3257a7, entries=150, sequenceid=411, filesize=30.5 K 2024-11-19T00:16:13,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/6b3cdad23bf8477bbb101e36149be251 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/6b3cdad23bf8477bbb101e36149be251 2024-11-19T00:16:13,836 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/6b3cdad23bf8477bbb101e36149be251, entries=150, sequenceid=411, filesize=12.0 K 2024-11-19T00:16:13,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/898ac263ede64076893d7174b1b6604b as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/898ac263ede64076893d7174b1b6604b 2024-11-19T00:16:13,841 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/898ac263ede64076893d7174b1b6604b, entries=150, sequenceid=411, filesize=12.0 K 2024-11-19T00:16:13,842 INFO [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=33.54 KB/34350 for cfb45e8932b673e843a8031625dfee57 in 1658ms, sequenceid=411, compaction requested=false 2024-11-19T00:16:13,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for cfb45e8932b673e843a8031625dfee57: 2024-11-19T00:16:13,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:13,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6b1531871d35:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-19T00:16:13,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-19T00:16:13,845 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-19T00:16:13,845 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0360 sec 2024-11-19T00:16:13,847 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 3.0400 sec 2024-11-19T00:16:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T00:16:14,965 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:16:14,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T00:16:14,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 107 2024-11-19T00:16:14,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-19T00:16:14,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-19T00:16:14,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-19T00:16:14,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-19T00:16:14,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T00:16:14,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2947 2024-11-19T00:16:14,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2926 2024-11-19T00:16:14,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2846 2024-11-19T00:16:14,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2949 2024-11-19T00:16:14,966 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2906 2024-11-19T00:16:14,966 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T00:16:14,966 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T00:16:14,966 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T00:16:14,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:14,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:14,966 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:16:14,967 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T00:16:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T00:16:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T00:16:14,973 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975374972"}]},"ts":"1731975374972"} 2024-11-19T00:16:14,974 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T00:16:14,974 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T00:16:14,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T00:16:14,976 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfb45e8932b673e843a8031625dfee57, UNASSIGN}] 2024-11-19T00:16:14,977 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfb45e8932b673e843a8031625dfee57, UNASSIGN 2024-11-19T00:16:14,978 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=cfb45e8932b673e843a8031625dfee57, regionState=CLOSING, regionLocation=6b1531871d35,38567,1731975194317 2024-11-19T00:16:14,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfb45e8932b673e843a8031625dfee57, UNASSIGN because future has completed 2024-11-19T00:16:14,979 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T00:16:14,979 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317}] 2024-11-19T00:16:15,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T00:16:15,131 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(122): Close cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:15,131 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T00:16:15,131 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1722): Closing cfb45e8932b673e843a8031625dfee57, disabling compactions & flushes 2024-11-19T00:16:15,131 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:15,131 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:15,131 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. after waiting 0 ms 2024-11-19T00:16:15,131 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:15,132 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(2902): Flushing cfb45e8932b673e843a8031625dfee57 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T00:16:15,132 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=A 2024-11-19T00:16:15,132 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:15,132 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=B 2024-11-19T00:16:15,132 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:15,132 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb45e8932b673e843a8031625dfee57, store=C 2024-11-19T00:16:15,132 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T00:16:15,138 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111998a2c1c9a2c742d28f263fc223f9b1aa_cfb45e8932b673e843a8031625dfee57 is 50, key is test_row_0/A:col10/1731975372534/Put/seqid=0 2024-11-19T00:16:15,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742474_1650 (size=12454) 2024-11-19T00:16:15,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T00:16:15,544 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T00:16:15,549 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111998a2c1c9a2c742d28f263fc223f9b1aa_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111998a2c1c9a2c742d28f263fc223f9b1aa_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:15,549 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/634f2d29f6c84a07bd387401140cd918, store: [table=TestAcidGuarantees family=A region=cfb45e8932b673e843a8031625dfee57] 2024-11-19T00:16:15,550 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/634f2d29f6c84a07bd387401140cd918 is 175, key is test_row_0/A:col10/1731975372534/Put/seqid=0 2024-11-19T00:16:15,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742475_1651 (size=31255) 2024-11-19T00:16:15,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T00:16:15,954 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=422, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/634f2d29f6c84a07bd387401140cd918 2024-11-19T00:16:15,964 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/bf5e0d752f504d5cb383824ab279d296 is 50, key is test_row_0/B:col10/1731975372534/Put/seqid=0 2024-11-19T00:16:15,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742476_1652 (size=12301) 2024-11-19T00:16:15,968 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/bf5e0d752f504d5cb383824ab279d296 2024-11-19T00:16:15,974 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/f6568966dc454b11ae5506022de4073e is 50, key is test_row_0/C:col10/1731975372534/Put/seqid=0 2024-11-19T00:16:15,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742477_1653 (size=12301) 2024-11-19T00:16:16,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T00:16:16,379 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/f6568966dc454b11ae5506022de4073e 2024-11-19T00:16:16,386 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/A/634f2d29f6c84a07bd387401140cd918 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/634f2d29f6c84a07bd387401140cd918 2024-11-19T00:16:16,390 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/634f2d29f6c84a07bd387401140cd918, entries=150, sequenceid=422, filesize=30.5 K 2024-11-19T00:16:16,391 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/B/bf5e0d752f504d5cb383824ab279d296 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/bf5e0d752f504d5cb383824ab279d296 2024-11-19T00:16:16,395 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/bf5e0d752f504d5cb383824ab279d296, entries=150, sequenceid=422, filesize=12.0 K 2024-11-19T00:16:16,395 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/.tmp/C/f6568966dc454b11ae5506022de4073e as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/f6568966dc454b11ae5506022de4073e 2024-11-19T00:16:16,398 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/f6568966dc454b11ae5506022de4073e, entries=150, sequenceid=422, filesize=12.0 K 2024-11-19T00:16:16,399 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for cfb45e8932b673e843a8031625dfee57 in 1268ms, sequenceid=422, compaction requested=true 2024-11-19T00:16:16,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/c6702a6160094448b3ae3cc863a651a3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/6527b32b8058494697649eb7b867a717, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/e219f363fdda48f289066a2f43e7d0c4, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/1e7c0d1525a74bd5b74b792c086f967d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/8f4d0a52941a40a79a4f02504077e6db, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/4c82092a39774e4791b9ff0a1ad9c097, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/c0d79232a028430aa0bb8a433f1b8e1e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/3910737fac7f48528da1388c527ab64c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/476e038d3ec8410e9a7952eeda08d1c5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/806eb1ca16e541ffa05e0e9fc1acfbc2, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/0868cbb3653043d68206ed8f5016571f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/af7b6340f3434a7a8aa454da4d62c86c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/94d0d2ccb9d944c8a485570d9109033b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/20b731532e4247948d1d2be50f85bd1e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/65b573d5b8a849d9bdb1581b2b0e2dd6, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/6e2ce54509094c55ae88f0a839bf5264, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/4ac24552e6424f2cacb88f29fda255e7, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/172aa14176f146d28175cd31234488e4, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/eca4f8aeff554c52ab01fb0fe2a22643, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/a1c21c0a72a24351b8c3b4124d847160, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/c252ac79bf874cb8b4b444067a907927, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/587f962bfce34088b3eb7dec58781a7c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/f1cd6a485b434703ab069236499ce6e5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/f82fbabdb0134a828c9159d847fd561d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/a5a657640ec34e6b99959a7ab7b5dd11] to archive 2024-11-19T00:16:16,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T00:16:16,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/c6702a6160094448b3ae3cc863a651a3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/c6702a6160094448b3ae3cc863a651a3 2024-11-19T00:16:16,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/6527b32b8058494697649eb7b867a717 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/6527b32b8058494697649eb7b867a717 2024-11-19T00:16:16,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/e219f363fdda48f289066a2f43e7d0c4 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/e219f363fdda48f289066a2f43e7d0c4 2024-11-19T00:16:16,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/1e7c0d1525a74bd5b74b792c086f967d to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/1e7c0d1525a74bd5b74b792c086f967d 2024-11-19T00:16:16,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/8f4d0a52941a40a79a4f02504077e6db to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/8f4d0a52941a40a79a4f02504077e6db 2024-11-19T00:16:16,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/4c82092a39774e4791b9ff0a1ad9c097 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/4c82092a39774e4791b9ff0a1ad9c097 2024-11-19T00:16:16,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/c0d79232a028430aa0bb8a433f1b8e1e to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/c0d79232a028430aa0bb8a433f1b8e1e 2024-11-19T00:16:16,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/3910737fac7f48528da1388c527ab64c to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/3910737fac7f48528da1388c527ab64c 2024-11-19T00:16:16,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/476e038d3ec8410e9a7952eeda08d1c5 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/476e038d3ec8410e9a7952eeda08d1c5 2024-11-19T00:16:16,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/806eb1ca16e541ffa05e0e9fc1acfbc2 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/806eb1ca16e541ffa05e0e9fc1acfbc2 2024-11-19T00:16:16,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/0868cbb3653043d68206ed8f5016571f to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/0868cbb3653043d68206ed8f5016571f 2024-11-19T00:16:16,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/af7b6340f3434a7a8aa454da4d62c86c to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/af7b6340f3434a7a8aa454da4d62c86c 2024-11-19T00:16:16,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/94d0d2ccb9d944c8a485570d9109033b to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/94d0d2ccb9d944c8a485570d9109033b 2024-11-19T00:16:16,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/20b731532e4247948d1d2be50f85bd1e to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/20b731532e4247948d1d2be50f85bd1e 2024-11-19T00:16:16,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/65b573d5b8a849d9bdb1581b2b0e2dd6 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/65b573d5b8a849d9bdb1581b2b0e2dd6 2024-11-19T00:16:16,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/6e2ce54509094c55ae88f0a839bf5264 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/6e2ce54509094c55ae88f0a839bf5264 2024-11-19T00:16:16,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/4ac24552e6424f2cacb88f29fda255e7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/4ac24552e6424f2cacb88f29fda255e7 2024-11-19T00:16:16,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/172aa14176f146d28175cd31234488e4 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/172aa14176f146d28175cd31234488e4 2024-11-19T00:16:16,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/eca4f8aeff554c52ab01fb0fe2a22643 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/eca4f8aeff554c52ab01fb0fe2a22643 2024-11-19T00:16:16,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/a1c21c0a72a24351b8c3b4124d847160 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/a1c21c0a72a24351b8c3b4124d847160 2024-11-19T00:16:16,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/c252ac79bf874cb8b4b444067a907927 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/c252ac79bf874cb8b4b444067a907927 2024-11-19T00:16:16,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/587f962bfce34088b3eb7dec58781a7c to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/587f962bfce34088b3eb7dec58781a7c 2024-11-19T00:16:16,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/f1cd6a485b434703ab069236499ce6e5 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/f1cd6a485b434703ab069236499ce6e5 2024-11-19T00:16:16,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/f82fbabdb0134a828c9159d847fd561d to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/f82fbabdb0134a828c9159d847fd561d 2024-11-19T00:16:16,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/a5a657640ec34e6b99959a7ab7b5dd11 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/a5a657640ec34e6b99959a7ab7b5dd11 2024-11-19T00:16:16,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/e57ebb7dd37a4a5ab3cd2c9c78a5be44, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/caef8f6b5f2249738618063c12dfec31, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/2db83489d4364853ac02c9c14beda29d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/f16e70a408784f58abe2c46a2a294d57, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/4467b17fa645498f99303e590e0ccae1, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/f41640f6e4dd446fbdbab1488b3241fe, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/e335deb159984ffa9f0ddcdeefceb734, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/1886c52eb0744db68fd932ba3314096d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/89f4893c78f04a71a2821a1ffe3c5635, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/5d1b46bacce445a8a8c94b4f6fbd454d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/b7a63f8628264a2b87e0be1bcccadaaf, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/26ae2b53980c4689ad02d099da2dc059, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/3f6d723d30e448a6b679d9798f3d9c70, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/15ee976c922d4f4699393262d5bc301d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/bc2914d8095e437296608d40206839fd, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/f4adcbf15109448d8420c2e11840723f, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/f4386f513cc4424fb51c76c13e2e7769, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/24ef9eb37f5f48b1aafc1694a9590328, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/d6d251cd804a4b7c89b7043cb7a55a5c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/215a2fabf91e43359c971bf8f0ba2238, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/59da562195f64d05a850495d7e0eb22e, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/c8b053b28fad42d780eb6201797e33c1, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/364794a9e65b49efa8d1509396a7bfd0, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/d2087119c5e845d1bef9d84b62e84be4, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/1385757cd20344829e48c9cdd7bc4c7b] to archive 2024-11-19T00:16:16,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T00:16:16,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/e57ebb7dd37a4a5ab3cd2c9c78a5be44 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/e57ebb7dd37a4a5ab3cd2c9c78a5be44 2024-11-19T00:16:16,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/caef8f6b5f2249738618063c12dfec31 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/caef8f6b5f2249738618063c12dfec31 2024-11-19T00:16:16,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/2db83489d4364853ac02c9c14beda29d to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/2db83489d4364853ac02c9c14beda29d 2024-11-19T00:16:16,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/f16e70a408784f58abe2c46a2a294d57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/f16e70a408784f58abe2c46a2a294d57 2024-11-19T00:16:16,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/4467b17fa645498f99303e590e0ccae1 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/4467b17fa645498f99303e590e0ccae1 2024-11-19T00:16:16,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/f41640f6e4dd446fbdbab1488b3241fe to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/f41640f6e4dd446fbdbab1488b3241fe 2024-11-19T00:16:16,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/e335deb159984ffa9f0ddcdeefceb734 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/e335deb159984ffa9f0ddcdeefceb734 2024-11-19T00:16:16,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/1886c52eb0744db68fd932ba3314096d to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/1886c52eb0744db68fd932ba3314096d 2024-11-19T00:16:16,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/89f4893c78f04a71a2821a1ffe3c5635 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/89f4893c78f04a71a2821a1ffe3c5635 2024-11-19T00:16:16,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/5d1b46bacce445a8a8c94b4f6fbd454d to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/5d1b46bacce445a8a8c94b4f6fbd454d 2024-11-19T00:16:16,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/b7a63f8628264a2b87e0be1bcccadaaf to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/b7a63f8628264a2b87e0be1bcccadaaf 2024-11-19T00:16:16,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/26ae2b53980c4689ad02d099da2dc059 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/26ae2b53980c4689ad02d099da2dc059 2024-11-19T00:16:16,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/3f6d723d30e448a6b679d9798f3d9c70 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/3f6d723d30e448a6b679d9798f3d9c70 2024-11-19T00:16:16,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/15ee976c922d4f4699393262d5bc301d to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/15ee976c922d4f4699393262d5bc301d 2024-11-19T00:16:16,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/bc2914d8095e437296608d40206839fd to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/bc2914d8095e437296608d40206839fd 2024-11-19T00:16:16,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/f4adcbf15109448d8420c2e11840723f to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/f4adcbf15109448d8420c2e11840723f 2024-11-19T00:16:16,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/f4386f513cc4424fb51c76c13e2e7769 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/f4386f513cc4424fb51c76c13e2e7769 2024-11-19T00:16:16,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/24ef9eb37f5f48b1aafc1694a9590328 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/24ef9eb37f5f48b1aafc1694a9590328 2024-11-19T00:16:16,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/d6d251cd804a4b7c89b7043cb7a55a5c to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/d6d251cd804a4b7c89b7043cb7a55a5c 2024-11-19T00:16:16,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/215a2fabf91e43359c971bf8f0ba2238 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/215a2fabf91e43359c971bf8f0ba2238 2024-11-19T00:16:16,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/59da562195f64d05a850495d7e0eb22e to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/59da562195f64d05a850495d7e0eb22e 2024-11-19T00:16:16,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/c8b053b28fad42d780eb6201797e33c1 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/c8b053b28fad42d780eb6201797e33c1 2024-11-19T00:16:16,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/364794a9e65b49efa8d1509396a7bfd0 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/364794a9e65b49efa8d1509396a7bfd0 2024-11-19T00:16:16,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/d2087119c5e845d1bef9d84b62e84be4 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/d2087119c5e845d1bef9d84b62e84be4 2024-11-19T00:16:16,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/1385757cd20344829e48c9cdd7bc4c7b to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/1385757cd20344829e48c9cdd7bc4c7b 2024-11-19T00:16:16,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/517f0ffcf49f48f5840ba79b8dc9ab27, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/98cbbb1caa6f432ab46a6325ee16453c, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/caf69cfd321341de890c9d94dc88b14b, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/06f285967b1846bd9f03d9f788dec337, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/3161c6d98a264b9a94c2841ac10bd340, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/52fd3c6c866749b586bea3441f3b4222, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/ffb8bd9d0211451b8076d447539acc93, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/703526b5b1534d11a21ec161bd938781, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/1ea3196725c74a43b7a7d2766b6d112d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/2e052dc056374c4f80c339ab170b147d, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/5501a76530fa4b458f9000781ab95f95, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/879f934daa6f4b5cb1a1fdb3758c0c25, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/98e0207a41bf44c3af8ee5b57d2d7cc5, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/e521383ecb614161a80d76d045eee301, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/b7b3366e21114c399ab385244fdcbaf3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/379d2fab38d04f8d8327966f5b8ea3ac, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/602baf8fc78d4863b1f3c6c225e65be9, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/cdc4aa61191a4f0088b871626faa2a44, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/c9f29dd8cdf34276ad7166390bbf779a, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/5c32fd5630d14252b790e98cefe3f962, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/d936f29648df473a9280bc7f8eb0e6b9, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/afcba2ddc6fd4d028e59faabd22a7c07, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/79bd722d9e9e4ebc92249673a897e9d3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/be02722038c347f782c799a6ceb4bac3, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/40241a3218134de793e7dee2fa01453d] to archive 2024-11-19T00:16:16,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T00:16:16,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/517f0ffcf49f48f5840ba79b8dc9ab27 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/517f0ffcf49f48f5840ba79b8dc9ab27 2024-11-19T00:16:16,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/98cbbb1caa6f432ab46a6325ee16453c to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/98cbbb1caa6f432ab46a6325ee16453c 2024-11-19T00:16:16,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/caf69cfd321341de890c9d94dc88b14b to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/caf69cfd321341de890c9d94dc88b14b 2024-11-19T00:16:16,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/06f285967b1846bd9f03d9f788dec337 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/06f285967b1846bd9f03d9f788dec337 2024-11-19T00:16:16,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/3161c6d98a264b9a94c2841ac10bd340 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/3161c6d98a264b9a94c2841ac10bd340 2024-11-19T00:16:16,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/52fd3c6c866749b586bea3441f3b4222 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/52fd3c6c866749b586bea3441f3b4222 2024-11-19T00:16:16,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/ffb8bd9d0211451b8076d447539acc93 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/ffb8bd9d0211451b8076d447539acc93 2024-11-19T00:16:16,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/703526b5b1534d11a21ec161bd938781 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/703526b5b1534d11a21ec161bd938781 2024-11-19T00:16:16,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/1ea3196725c74a43b7a7d2766b6d112d to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/1ea3196725c74a43b7a7d2766b6d112d 2024-11-19T00:16:16,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/2e052dc056374c4f80c339ab170b147d to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/2e052dc056374c4f80c339ab170b147d 2024-11-19T00:16:16,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/5501a76530fa4b458f9000781ab95f95 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/5501a76530fa4b458f9000781ab95f95 2024-11-19T00:16:16,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/879f934daa6f4b5cb1a1fdb3758c0c25 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/879f934daa6f4b5cb1a1fdb3758c0c25 2024-11-19T00:16:16,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/98e0207a41bf44c3af8ee5b57d2d7cc5 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/98e0207a41bf44c3af8ee5b57d2d7cc5 2024-11-19T00:16:16,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/e521383ecb614161a80d76d045eee301 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/e521383ecb614161a80d76d045eee301 2024-11-19T00:16:16,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/b7b3366e21114c399ab385244fdcbaf3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/b7b3366e21114c399ab385244fdcbaf3 2024-11-19T00:16:16,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/379d2fab38d04f8d8327966f5b8ea3ac to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/379d2fab38d04f8d8327966f5b8ea3ac 2024-11-19T00:16:16,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/602baf8fc78d4863b1f3c6c225e65be9 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/602baf8fc78d4863b1f3c6c225e65be9 2024-11-19T00:16:16,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/cdc4aa61191a4f0088b871626faa2a44 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/cdc4aa61191a4f0088b871626faa2a44 2024-11-19T00:16:16,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/c9f29dd8cdf34276ad7166390bbf779a to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/c9f29dd8cdf34276ad7166390bbf779a 2024-11-19T00:16:16,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/5c32fd5630d14252b790e98cefe3f962 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/5c32fd5630d14252b790e98cefe3f962 2024-11-19T00:16:16,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/d936f29648df473a9280bc7f8eb0e6b9 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/d936f29648df473a9280bc7f8eb0e6b9 2024-11-19T00:16:16,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/afcba2ddc6fd4d028e59faabd22a7c07 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/afcba2ddc6fd4d028e59faabd22a7c07 2024-11-19T00:16:16,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/79bd722d9e9e4ebc92249673a897e9d3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/79bd722d9e9e4ebc92249673a897e9d3 2024-11-19T00:16:16,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/be02722038c347f782c799a6ceb4bac3 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/be02722038c347f782c799a6ceb4bac3 2024-11-19T00:16:16,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/40241a3218134de793e7dee2fa01453d to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/40241a3218134de793e7dee2fa01453d 2024-11-19T00:16:16,512 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/recovered.edits/425.seqid, newMaxSeqId=425, maxSeqId=4 2024-11-19T00:16:16,513 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. 2024-11-19T00:16:16,513 DEBUG [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1676): Region close journal for cfb45e8932b673e843a8031625dfee57: Waiting for close lock at 1731975375131Running coprocessor pre-close hooks at 1731975375131Disabling compacts and flushes for region at 1731975375131Disabling writes for close at 1731975375131Obtaining lock to block concurrent updates at 1731975375132 (+1 ms)Preparing flush snapshotting stores in cfb45e8932b673e843a8031625dfee57 at 1731975375132Finished memstore snapshotting TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731975375132Flushing stores of TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57. at 1731975375132Flushing cfb45e8932b673e843a8031625dfee57/A: creating writer at 1731975375132Flushing mob file cfb45e8932b673e843a8031625dfee57/A: appending metadata at 1731975375138 (+6 ms)Flushing mob file cfb45e8932b673e843a8031625dfee57/A: closing flushed file at 1731975375138Flushing cfb45e8932b673e843a8031625dfee57/A: appending metadata at 1731975375549 (+411 ms)Flushing cfb45e8932b673e843a8031625dfee57/A: closing flushed file at 1731975375549Flushing cfb45e8932b673e843a8031625dfee57/B: creating writer at 1731975375959 (+410 ms)Flushing cfb45e8932b673e843a8031625dfee57/B: appending metadata at 1731975375964 (+5 ms)Flushing cfb45e8932b673e843a8031625dfee57/B: closing flushed file at 1731975375964Flushing cfb45e8932b673e843a8031625dfee57/C: creating writer at 1731975375971 (+7 ms)Flushing cfb45e8932b673e843a8031625dfee57/C: appending metadata at 1731975375974 (+3 ms)Flushing cfb45e8932b673e843a8031625dfee57/C: closing flushed file at 1731975375974Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31664522: reopening flushed file at 1731975376385 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bf44176: reopening flushed file at 1731975376391 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@522ad77f: reopening flushed file at 1731975376395 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for cfb45e8932b673e843a8031625dfee57 in 1268ms, sequenceid=422, compaction requested=true at 1731975376399 (+4 ms)Writing region close event to WAL at 1731975376510 (+111 ms)Running coprocessor post-close hooks at 1731975376513 (+3 ms)Closed at 1731975376513 2024-11-19T00:16:16,515 INFO [RS_CLOSE_REGION-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(157): Closed cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:16,515 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=cfb45e8932b673e843a8031625dfee57, regionState=CLOSED 2024-11-19T00:16:16,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 because future has completed 2024-11-19T00:16:16,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-19T00:16:16,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseRegionProcedure cfb45e8932b673e843a8031625dfee57, server=6b1531871d35,38567,1731975194317 in 1.5380 sec 2024-11-19T00:16:16,520 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=162, resume processing ppid=161 2024-11-19T00:16:16,520 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, ppid=161, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfb45e8932b673e843a8031625dfee57, UNASSIGN in 1.5420 sec 2024-11-19T00:16:16,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-19T00:16:16,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5450 sec 2024-11-19T00:16:16,523 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731975376523"}]},"ts":"1731975376523"} 2024-11-19T00:16:16,525 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T00:16:16,525 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T00:16:16,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5590 sec 2024-11-19T00:16:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T00:16:17,114 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:16:17,114 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T00:16:17,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:16:17,116 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:16:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T00:16:17,117 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=164, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:16:17,118 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:17,120 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A, FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B, FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C, FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/recovered.edits] 2024-11-19T00:16:17,124 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/634f2d29f6c84a07bd387401140cd918 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/634f2d29f6c84a07bd387401140cd918 2024-11-19T00:16:17,125 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/8f15a3092c564cd7b3143016fc3257a7 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/8f15a3092c564cd7b3143016fc3257a7 2024-11-19T00:16:17,126 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/e78e9b0675854607b192d50c753fe080 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/A/e78e9b0675854607b192d50c753fe080 2024-11-19T00:16:17,128 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/6b3cdad23bf8477bbb101e36149be251 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/6b3cdad23bf8477bbb101e36149be251 2024-11-19T00:16:17,130 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/bf5e0d752f504d5cb383824ab279d296 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/bf5e0d752f504d5cb383824ab279d296 2024-11-19T00:16:17,131 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/e07d94477aaf4029a545cae31a00203b to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/B/e07d94477aaf4029a545cae31a00203b 2024-11-19T00:16:17,133 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/1a9c1cf7496440e5bb1f58b709d9fb8d to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/1a9c1cf7496440e5bb1f58b709d9fb8d 2024-11-19T00:16:17,134 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/898ac263ede64076893d7174b1b6604b to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/898ac263ede64076893d7174b1b6604b 2024-11-19T00:16:17,135 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/f6568966dc454b11ae5506022de4073e to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/C/f6568966dc454b11ae5506022de4073e 2024-11-19T00:16:17,138 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/recovered.edits/425.seqid to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57/recovered.edits/425.seqid 2024-11-19T00:16:17,138 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/default/TestAcidGuarantees/cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:17,138 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T00:16:17,139 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T00:16:17,140 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T00:16:17,143 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190217b31bd1c348d5944b8c7bf85d5c7c_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190217b31bd1c348d5944b8c7bf85d5c7c_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:17,144 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111923cd5c4cf2df4f7eacb19e91f2d3a477_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111923cd5c4cf2df4f7eacb19e91f2d3a477_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:17,145 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192aba74633d6843658aafa8494ca2be2b_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192aba74633d6843658aafa8494ca2be2b_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:17,147 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193d8b71214ef1491394db7a8bb6f4eea9_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193d8b71214ef1491394db7a8bb6f4eea9_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:17,148 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119426c084a08a84cd8a342a34857ed916f_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119426c084a08a84cd8a342a34857ed916f_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:17,149 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196b868475e83c4f5aa2e3beb96c9622b4_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196b868475e83c4f5aa2e3beb96c9622b4_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:17,150 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197e8e58df2fab49f8bd65608e996e0f97_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197e8e58df2fab49f8bd65608e996e0f97_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:17,151 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119830e7ffdfe844b4c9b461713f8daa98a_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119830e7ffdfe844b4c9b461713f8daa98a_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:17,152 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198fd1805e34cd4c2ab5b0089beb16673c_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198fd1805e34cd4c2ab5b0089beb16673c_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:17,153 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119938fa31dc976423bb76a3896226e1de4_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119938fa31dc976423bb76a3896226e1de4_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:17,154 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111998a2c1c9a2c742d28f263fc223f9b1aa_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111998a2c1c9a2c742d28f263fc223f9b1aa_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:17,155 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199c74d37128914710a96771d920d95491_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199c74d37128914710a96771d920d95491_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:17,156 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199cd8e46276454eb58da1f5c4d0a3fbe9_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199cd8e46276454eb58da1f5c4d0a3fbe9_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:17,157 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ac52e0a34cc04eaaa0c94dfaf9b1206d_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ac52e0a34cc04eaaa0c94dfaf9b1206d_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:17,158 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c2a881cf354c434682dc9b632cc02595_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c2a881cf354c434682dc9b632cc02595_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:17,159 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d2387017a8bd4e6297801596781a1f5b_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d2387017a8bd4e6297801596781a1f5b_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:17,160 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d6e45c5fc64145858fb07fe802d4e98e_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d6e45c5fc64145858fb07fe802d4e98e_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:17,161 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ddb17f701c4c4055b228930804747d4e_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ddb17f701c4c4055b228930804747d4e_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:17,163 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dee7de1855034098be7ce07f92b96f3e_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dee7de1855034098be7ce07f92b96f3e_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:17,164 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e5924839d45f4f479b134f47438a2f53_cfb45e8932b673e843a8031625dfee57 to hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e5924839d45f4f479b134f47438a2f53_cfb45e8932b673e843a8031625dfee57 2024-11-19T00:16:17,164 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T00:16:17,167 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=164, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:16:17,170 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T00:16:17,173 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T00:16:17,174 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=164, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:16:17,174 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T00:16:17,175 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731975377174"}]},"ts":"9223372036854775807"} 2024-11-19T00:16:17,177 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T00:16:17,177 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cfb45e8932b673e843a8031625dfee57, NAME => 'TestAcidGuarantees,,1731975348953.cfb45e8932b673e843a8031625dfee57.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T00:16:17,177 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T00:16:17,178 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731975377178"}]},"ts":"9223372036854775807"} 2024-11-19T00:16:17,180 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T00:16:17,181 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=164, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T00:16:17,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 66 msec 2024-11-19T00:16:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T00:16:17,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T00:16:17,224 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T00:16:17,235 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250 (was 252), OpenFileDescriptor=451 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=604 (was 492) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1756 (was 1341) - AvailableMemoryMB LEAK? - 2024-11-19T00:16:17,236 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-19T00:16:17,236 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T00:16:17,236 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T00:16:17,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:17,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:17,236 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T00:16:17,236 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-19T00:16:17,236 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1219616701, stopped=false 2024-11-19T00:16:17,237 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=6b1531871d35,33843,1731975193570 2024-11-19T00:16:17,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38567-0x10055461f5a0001, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T00:16:17,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38567-0x10055461f5a0001, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:16:17,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T00:16:17,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:16:17,431 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T00:16:17,432 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T00:16:17,432 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T00:16:17,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:17,432 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38567-0x10055461f5a0001, quorum=127.0.0.1:62652, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T00:16:17,432 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6b1531871d35,38567,1731975194317' ***** 2024-11-19T00:16:17,432 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T00:16:17,432 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T00:16:17,433 INFO [RS:0;6b1531871d35:38567 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T00:16:17,433 INFO [RS:0;6b1531871d35:38567 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T00:16:17,433 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T00:16:17,433 INFO [RS:0;6b1531871d35:38567 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T00:16:17,433 INFO [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(959): stopping server 6b1531871d35,38567,1731975194317 2024-11-19T00:16:17,433 INFO [RS:0;6b1531871d35:38567 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T00:16:17,433 INFO [RS:0;6b1531871d35:38567 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;6b1531871d35:38567. 2024-11-19T00:16:17,433 DEBUG [RS:0;6b1531871d35:38567 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T00:16:17,433 DEBUG [RS:0;6b1531871d35:38567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:17,434 INFO [RS:0;6b1531871d35:38567 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T00:16:17,434 INFO [RS:0;6b1531871d35:38567 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T00:16:17,434 INFO [RS:0;6b1531871d35:38567 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T00:16:17,434 INFO [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-19T00:16:17,434 INFO [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-19T00:16:17,434 DEBUG [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-19T00:16:17,435 DEBUG [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T00:16:17,435 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T00:16:17,435 INFO [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T00:16:17,435 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T00:16:17,435 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T00:16:17,435 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T00:16:17,435 INFO [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-19T00:16:17,458 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740/.tmp/info/17c0b8b27edf4a2ca0f2d5bc391f71dc is 95, key is TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a./info:/1731975226750/DeleteFamily/seqid=0 2024-11-19T00:16:17,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742478_1654 (size=6514) 2024-11-19T00:16:17,474 INFO [regionserver/6b1531871d35:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T00:16:17,635 DEBUG [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T00:16:17,835 DEBUG [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T00:16:17,862 INFO [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740/.tmp/info/17c0b8b27edf4a2ca0f2d5bc391f71dc 2024-11-19T00:16:17,895 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740/.tmp/ns/cfdb533be4c8429aaf6e3c486eccc8cb is 93, key is TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a./ns:/1731975226684/DeleteFamily/seqid=0 2024-11-19T00:16:17,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742479_1655 (size=5906) 2024-11-19T00:16:17,941 INFO [regionserver/6b1531871d35:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-19T00:16:17,941 INFO [regionserver/6b1531871d35:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-19T00:16:18,035 DEBUG [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T00:16:18,236 DEBUG [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T00:16:18,299 INFO [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740/.tmp/ns/cfdb533be4c8429aaf6e3c486eccc8cb 2024-11-19T00:16:18,327 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740/.tmp/rep_barrier/406f02e1a3ec4c4a95a53d082e9292fc is 102, key is TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a./rep_barrier:/1731975226684/DeleteFamily/seqid=0 2024-11-19T00:16:18,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742480_1656 (size=6025) 2024-11-19T00:16:18,436 DEBUG [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T00:16:18,636 DEBUG [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T00:16:18,732 INFO [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740/.tmp/rep_barrier/406f02e1a3ec4c4a95a53d082e9292fc 2024-11-19T00:16:18,758 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740/.tmp/table/4954b017947a4a63bb16341ce0633ba7 is 96, key is TestAcidGuarantees,,1731975198801.df30878ae408737da14ed71330503a0a./table:/1731975226684/DeleteFamily/seqid=0 2024-11-19T00:16:18,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742481_1657 (size=5936) 2024-11-19T00:16:18,836 DEBUG [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T00:16:19,037 DEBUG [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T00:16:19,162 INFO [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740/.tmp/table/4954b017947a4a63bb16341ce0633ba7 2024-11-19T00:16:19,165 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740/.tmp/info/17c0b8b27edf4a2ca0f2d5bc391f71dc as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740/info/17c0b8b27edf4a2ca0f2d5bc391f71dc 2024-11-19T00:16:19,168 INFO [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740/info/17c0b8b27edf4a2ca0f2d5bc391f71dc, entries=12, sequenceid=90, filesize=6.4 K 2024-11-19T00:16:19,169 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740/.tmp/ns/cfdb533be4c8429aaf6e3c486eccc8cb as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740/ns/cfdb533be4c8429aaf6e3c486eccc8cb 2024-11-19T00:16:19,172 INFO [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740/ns/cfdb533be4c8429aaf6e3c486eccc8cb, entries=8, sequenceid=90, filesize=5.8 K 2024-11-19T00:16:19,173 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740/.tmp/rep_barrier/406f02e1a3ec4c4a95a53d082e9292fc as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740/rep_barrier/406f02e1a3ec4c4a95a53d082e9292fc 2024-11-19T00:16:19,175 INFO [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740/rep_barrier/406f02e1a3ec4c4a95a53d082e9292fc, entries=6, sequenceid=90, filesize=5.9 K 2024-11-19T00:16:19,176 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740/.tmp/table/4954b017947a4a63bb16341ce0633ba7 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740/table/4954b017947a4a63bb16341ce0633ba7 2024-11-19T00:16:19,179 INFO [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740/table/4954b017947a4a63bb16341ce0633ba7, entries=7, sequenceid=90, filesize=5.8 K 2024-11-19T00:16:19,180 INFO [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1745ms, sequenceid=90, compaction requested=false 2024-11-19T00:16:19,184 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-19T00:16:19,184 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T00:16:19,185 INFO [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T00:16:19,185 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731975377434Running coprocessor pre-close hooks at 1731975377435 (+1 ms)Disabling compacts and flushes for region at 1731975377435Disabling writes for close at 1731975377435Obtaining lock to block concurrent updates at 1731975377435Preparing flush snapshotting stores in 1588230740 at 1731975377435Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731975377435Flushing stores of hbase:meta,,1.1588230740 at 1731975377436 (+1 ms)Flushing 1588230740/info: creating writer at 1731975377436Flushing 1588230740/info: appending metadata at 1731975377458 (+22 ms)Flushing 1588230740/info: closing flushed file at 1731975377458Flushing 1588230740/ns: creating writer at 1731975377881 (+423 ms)Flushing 1588230740/ns: appending metadata at 1731975377895 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731975377895Flushing 1588230740/rep_barrier: creating writer at 1731975378308 (+413 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731975378327 (+19 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731975378327Flushing 1588230740/table: creating writer at 1731975378739 (+412 ms)Flushing 1588230740/table: appending metadata at 1731975378758 (+19 ms)Flushing 1588230740/table: closing flushed file at 1731975378758Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ac6635f: reopening flushed file at 1731975379165 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ab0ffe1: reopening flushed file at 1731975379169 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@194f16a0: reopening flushed file at 1731975379172 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a1cf1d1: reopening flushed file at 1731975379176 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1745ms, sequenceid=90, compaction requested=false at 1731975379180 (+4 ms)Writing region close event to WAL at 1731975379181 (+1 ms)Running coprocessor post-close hooks at 1731975379184 (+3 ms)Closed at 1731975379185 (+1 ms) 2024-11-19T00:16:19,185 DEBUG [RS_CLOSE_META-regionserver/6b1531871d35:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-19T00:16:19,237 INFO [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(976): stopping server 6b1531871d35,38567,1731975194317; all regions closed. 2024-11-19T00:16:19,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741834_1010 (size=25344) 2024-11-19T00:16:19,243 DEBUG [RS:0;6b1531871d35:38567 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/oldWALs 2024-11-19T00:16:19,243 INFO [RS:0;6b1531871d35:38567 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6b1531871d35%2C38567%2C1731975194317.meta:.meta(num 1731975198082) 2024-11-19T00:16:19,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741832_1008 (size=18631152) 2024-11-19T00:16:19,247 DEBUG [RS:0;6b1531871d35:38567 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/oldWALs 2024-11-19T00:16:19,247 INFO [RS:0;6b1531871d35:38567 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6b1531871d35%2C38567%2C1731975194317:(num 1731975197100) 2024-11-19T00:16:19,247 DEBUG [RS:0;6b1531871d35:38567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T00:16:19,247 INFO [RS:0;6b1531871d35:38567 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T00:16:19,247 INFO [RS:0;6b1531871d35:38567 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T00:16:19,248 INFO [RS:0;6b1531871d35:38567 {}] hbase.ChoreService(370): Chore service for: regionserver/6b1531871d35:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-19T00:16:19,248 INFO [RS:0;6b1531871d35:38567 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T00:16:19,248 INFO [regionserver/6b1531871d35:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T00:16:19,249 INFO [RS:0;6b1531871d35:38567 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38567 2024-11-19T00:16:19,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38567-0x10055461f5a0001, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6b1531871d35,38567,1731975194317 2024-11-19T00:16:19,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T00:16:19,290 INFO [RS:0;6b1531871d35:38567 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T00:16:19,298 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6b1531871d35,38567,1731975194317] 2024-11-19T00:16:19,306 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6b1531871d35,38567,1731975194317 already deleted, retry=false 2024-11-19T00:16:19,306 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6b1531871d35,38567,1731975194317 expired; onlineServers=0 2024-11-19T00:16:19,306 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '6b1531871d35,33843,1731975193570' ***** 2024-11-19T00:16:19,306 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-19T00:16:19,307 INFO [M:0;6b1531871d35:33843 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T00:16:19,307 INFO [M:0;6b1531871d35:33843 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T00:16:19,307 DEBUG [M:0;6b1531871d35:33843 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-19T00:16:19,307 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-19T00:16:19,307 DEBUG [M:0;6b1531871d35:33843 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-19T00:16:19,307 DEBUG [master/6b1531871d35:0:becomeActiveMaster-HFileCleaner.large.0-1731975196747 {}] cleaner.HFileCleaner(306): Exit Thread[master/6b1531871d35:0:becomeActiveMaster-HFileCleaner.large.0-1731975196747,5,FailOnTimeoutGroup] 2024-11-19T00:16:19,307 DEBUG [master/6b1531871d35:0:becomeActiveMaster-HFileCleaner.small.0-1731975196747 {}] cleaner.HFileCleaner(306): Exit Thread[master/6b1531871d35:0:becomeActiveMaster-HFileCleaner.small.0-1731975196747,5,FailOnTimeoutGroup] 2024-11-19T00:16:19,307 INFO [M:0;6b1531871d35:33843 {}] hbase.ChoreService(370): Chore service for: master/6b1531871d35:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-19T00:16:19,307 INFO [M:0;6b1531871d35:33843 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T00:16:19,307 DEBUG [M:0;6b1531871d35:33843 {}] master.HMaster(1795): Stopping service threads 2024-11-19T00:16:19,307 INFO [M:0;6b1531871d35:33843 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-19T00:16:19,307 INFO [M:0;6b1531871d35:33843 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T00:16:19,308 ERROR [M:0;6b1531871d35:33843 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-19T00:16:19,308 INFO [M:0;6b1531871d35:33843 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T00:16:19,308 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-19T00:16:19,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-19T00:16:19,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T00:16:19,318 DEBUG [M:0;6b1531871d35:33843 {}] zookeeper.ZKUtil(347): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-19T00:16:19,318 WARN [M:0;6b1531871d35:33843 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-19T00:16:19,319 INFO [M:0;6b1531871d35:33843 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/.lastflushedseqids 2024-11-19T00:16:19,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742482_1658 (size=72) 2024-11-19T00:16:19,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38567-0x10055461f5a0001, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T00:16:19,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38567-0x10055461f5a0001, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T00:16:19,399 INFO [RS:0;6b1531871d35:38567 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T00:16:19,399 INFO [RS:0;6b1531871d35:38567 {}] regionserver.HRegionServer(1031): Exiting; stopping=6b1531871d35,38567,1731975194317; zookeeper connection closed. 2024-11-19T00:16:19,399 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@473c6ce0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@473c6ce0 2024-11-19T00:16:19,400 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-19T00:16:19,730 INFO [M:0;6b1531871d35:33843 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T00:16:19,731 INFO [M:0;6b1531871d35:33843 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-19T00:16:19,731 DEBUG [M:0;6b1531871d35:33843 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T00:16:19,731 INFO [M:0;6b1531871d35:33843 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T00:16:19,731 DEBUG [M:0;6b1531871d35:33843 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T00:16:19,731 DEBUG [M:0;6b1531871d35:33843 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T00:16:19,731 DEBUG [M:0;6b1531871d35:33843 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T00:16:19,731 INFO [M:0;6b1531871d35:33843 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=761.13 KB heapSize=931.93 KB 2024-11-19T00:16:19,750 DEBUG [M:0;6b1531871d35:33843 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4277ec46fc4f4f81a1ea03600565dd7d is 82, key is hbase:meta,,1/info:regioninfo/1731975198222/Put/seqid=0 2024-11-19T00:16:19,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742483_1659 (size=5672) 2024-11-19T00:16:19,761 INFO [M:0;6b1531871d35:33843 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2109 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4277ec46fc4f4f81a1ea03600565dd7d 2024-11-19T00:16:19,793 DEBUG [M:0;6b1531871d35:33843 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0988adbe951649d585a728a704cf5fc9 is 2479, key is \x00\x00\x00\x00\x00\x00\x00X/proc:d/1731975295633/Put/seqid=0 2024-11-19T00:16:19,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742484_1660 (size=44298) 2024-11-19T00:16:19,803 INFO [M:0;6b1531871d35:33843 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=760.58 KB at sequenceid=2109 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0988adbe951649d585a728a704cf5fc9 2024-11-19T00:16:19,809 INFO [M:0;6b1531871d35:33843 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0988adbe951649d585a728a704cf5fc9 2024-11-19T00:16:19,832 DEBUG [M:0;6b1531871d35:33843 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f1f2dc3e97ad45f286f0fc95b4ef192f is 69, key is 6b1531871d35,38567,1731975194317/rs:state/1731975196863/Put/seqid=0 2024-11-19T00:16:19,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073742485_1661 (size=5156) 2024-11-19T00:16:20,239 INFO [M:0;6b1531871d35:33843 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2109 (bloomFilter=true), to=hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f1f2dc3e97ad45f286f0fc95b4ef192f 2024-11-19T00:16:20,244 DEBUG [M:0;6b1531871d35:33843 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4277ec46fc4f4f81a1ea03600565dd7d as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4277ec46fc4f4f81a1ea03600565dd7d 2024-11-19T00:16:20,248 INFO [M:0;6b1531871d35:33843 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4277ec46fc4f4f81a1ea03600565dd7d, entries=8, sequenceid=2109, filesize=5.5 K 2024-11-19T00:16:20,248 DEBUG [M:0;6b1531871d35:33843 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0988adbe951649d585a728a704cf5fc9 as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0988adbe951649d585a728a704cf5fc9 2024-11-19T00:16:20,252 INFO [M:0;6b1531871d35:33843 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0988adbe951649d585a728a704cf5fc9 2024-11-19T00:16:20,252 INFO [M:0;6b1531871d35:33843 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0988adbe951649d585a728a704cf5fc9, entries=164, sequenceid=2109, filesize=43.3 K 2024-11-19T00:16:20,253 DEBUG [M:0;6b1531871d35:33843 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f1f2dc3e97ad45f286f0fc95b4ef192f as hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f1f2dc3e97ad45f286f0fc95b4ef192f 2024-11-19T00:16:20,258 INFO [M:0;6b1531871d35:33843 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36755/user/jenkins/test-data/4366c7b3-0672-1c75-d5c7-30192a150ec2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f1f2dc3e97ad45f286f0fc95b4ef192f, entries=1, sequenceid=2109, filesize=5.0 K 2024-11-19T00:16:20,259 INFO [M:0;6b1531871d35:33843 {}] regionserver.HRegion(3140): Finished flush of dataSize ~761.13 KB/779401, heapSize ~931.63 KB/953992, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 527ms, sequenceid=2109, compaction requested=false 2024-11-19T00:16:20,275 INFO [M:0;6b1531871d35:33843 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T00:16:20,275 DEBUG [M:0;6b1531871d35:33843 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731975379731Disabling compacts and flushes for region at 1731975379731Disabling writes for close at 1731975379731Obtaining lock to block concurrent updates at 1731975379731Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731975379731Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=779401, getHeapSize=954232, getOffHeapSize=0, getCellsCount=2298 at 1731975379731Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731975379732 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731975379732Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731975379749 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731975379749Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731975379765 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731975379792 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731975379792Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731975379809 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731975379831 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731975379831Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62cb5c4d: reopening flushed file at 1731975380243 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37ba0fc9: reopening flushed file at 1731975380248 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@558f552e: reopening flushed file at 1731975380253 (+5 ms)Finished flush of dataSize ~761.13 KB/779401, heapSize ~931.63 KB/953992, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 527ms, sequenceid=2109, compaction requested=false at 1731975380259 (+6 ms)Writing region close event to WAL at 1731975380275 (+16 ms)Closed at 1731975380275 2024-11-19T00:16:20,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33587 is added to blk_1073741830_1006 (size=917661) 2024-11-19T00:16:20,280 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T00:16:20,281 INFO [M:0;6b1531871d35:33843 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-19T00:16:20,281 INFO [M:0;6b1531871d35:33843 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33843 2024-11-19T00:16:20,282 INFO [M:0;6b1531871d35:33843 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T00:16:20,433 INFO [M:0;6b1531871d35:33843 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T00:16:20,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T00:16:20,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33843-0x10055461f5a0000, quorum=127.0.0.1:62652, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T00:16:20,440 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T00:16:20,442 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T00:16:20,443 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T00:16:20,443 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T00:16:20,443 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/hadoop.log.dir/,STOPPED} 2024-11-19T00:16:20,446 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-19T00:16:20,446 WARN [BP-1176150126-172.17.0.2-1731975190061 heartbeating to localhost/127.0.0.1:36755 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-19T00:16:20,446 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-19T00:16:20,446 WARN [BP-1176150126-172.17.0.2-1731975190061 heartbeating to localhost/127.0.0.1:36755 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1176150126-172.17.0.2-1731975190061 (Datanode Uuid 7cb7ba0a-ea0f-4bdc-bc40-7cf0d89b01ba) service to localhost/127.0.0.1:36755 2024-11-19T00:16:20,450 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/cluster_ec2aa8be-d8f2-2fc7-fbd3-add83417a4c1/data/data1/current/BP-1176150126-172.17.0.2-1731975190061 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T00:16:20,450 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/cluster_ec2aa8be-d8f2-2fc7-fbd3-add83417a4c1/data/data2/current/BP-1176150126-172.17.0.2-1731975190061 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T00:16:20,450 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-19T00:16:20,459 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T00:16:20,460 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T00:16:20,460 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T00:16:20,460 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T00:16:20,460 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55325ff0-3e3f-5196-f6a5-b8251cd7c9c1/hadoop.log.dir/,STOPPED} 2024-11-19T00:16:20,476 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-19T00:16:20,649 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down